[robot] Add documentation of the robot suite
Add a documentation and license files for the test suite references as well as some tox setup files in case are needed to apply linters on the suite. Depends-On : Ia1e4e9ce011e1e527bd1109d1260bdc0b9a410b9 Signed-off-by: Jose Perez Carranza <jose.perez.carranza@intel.com> Change-Id: If81c702688a0cb3daed5c2cbe3181eb3fc6039d8
This commit is contained in:
parent
4fd5d5371b
commit
8da6f67057
|
@ -0,0 +1,2 @@
|
||||||
|
All source code id licensed under Apache 2.0 license unless otherwise stated
|
||||||
|
on the source file. See LICENSE.apache for further details.
|
|
@ -0,0 +1,202 @@
|
||||||
|
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
|
@ -0,0 +1,548 @@
|
||||||
|
.. default-role:: code
|
||||||
|
|
||||||
|
======================================================================
|
||||||
|
How to write good test cases using Robot Framework for stx-test-suite
|
||||||
|
======================================================================
|
||||||
|
|
||||||
|
.. contents:: Table of contents:
|
||||||
|
:local:
|
||||||
|
:depth: 2
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
============
|
||||||
|
|
||||||
|
- These are high-level guidelines for writing good test cases using Robot
|
||||||
|
Framework. It is based on the official `how to`__ file
|
||||||
|
for Robot Framework.
|
||||||
|
|
||||||
|
__ https://github.com/robotframework/HowToWriteGoodTestCases/blob/master/
|
||||||
|
HowToWriteGoodTestCases.rst
|
||||||
|
|
||||||
|
- How to actually interact with the system under test is out of
|
||||||
|
the scope of this document.
|
||||||
|
|
||||||
|
- Most important guideline is keeping test cases as easy to understand as
|
||||||
|
possible for people familiar with the domain.
|
||||||
|
|
||||||
|
- This typically also eases maintenance.
|
||||||
|
|
||||||
|
Naming
|
||||||
|
======
|
||||||
|
|
||||||
|
Test suite names
|
||||||
|
----------------
|
||||||
|
|
||||||
|
- Suite names should be as descriptive as possible.
|
||||||
|
|
||||||
|
- Names are created automatically from file or directory names:
|
||||||
|
|
||||||
|
- Extensions are stripped.
|
||||||
|
- Underscores are converted to spaces.
|
||||||
|
- If name is all lower case, words are capitalized.
|
||||||
|
|
||||||
|
- Names can be relatively long, but overly long names are not convenient for
|
||||||
|
the file system.
|
||||||
|
|
||||||
|
- The name of the top level suite can be overridden from the command line
|
||||||
|
using the `--name` option if needed.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- `login_tests.robot` -> `Login Tests`
|
||||||
|
- `IP_v4_and_v6` -> `IP v4 and v6`
|
||||||
|
|
||||||
|
|
||||||
|
Test case names
|
||||||
|
---------------
|
||||||
|
|
||||||
|
- Test names should be descriptive like the suite names.
|
||||||
|
|
||||||
|
- If a suite contains many similar tests and is well named,
|
||||||
|
test names can be shorter.
|
||||||
|
|
||||||
|
- Name is exactly the same as you specified in the test case file without any
|
||||||
|
conversion.
|
||||||
|
|
||||||
|
For example, if we have tests related to invalid login in a file
|
||||||
|
`invalid_login.robot`, these would be OK test case names:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Empty Password
|
||||||
|
Empty Username
|
||||||
|
Empty Username And Password
|
||||||
|
Invalid Username
|
||||||
|
Invalid Password
|
||||||
|
Invalid Username And Password
|
||||||
|
|
||||||
|
These names would be somewhat long:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Login With Empty Password Should Fail
|
||||||
|
Login With Empty Username Should Fail
|
||||||
|
Login With Empty Username And Password Should Fail
|
||||||
|
Login With Invalid Username Should Fail
|
||||||
|
Login With Invalid Password Should Fail
|
||||||
|
Login With Invalid Username And Invalid Password Should Fail
|
||||||
|
|
||||||
|
|
||||||
|
Keyword names
|
||||||
|
-------------
|
||||||
|
|
||||||
|
- Keyword names should be descriptive and clear.
|
||||||
|
|
||||||
|
- Should explain what the keyword does, not how it does its task(s).
|
||||||
|
|
||||||
|
- Very different abstraction levels (e.g. `Input Text` or `Administrator
|
||||||
|
logs into system`).
|
||||||
|
|
||||||
|
- There is no clear guideline on whether a keyword should be fully title cased
|
||||||
|
or have only the first letter be capitalized.
|
||||||
|
|
||||||
|
- Title casing is often used when the keyword name is short
|
||||||
|
(e.g. `Input Text`).
|
||||||
|
- Capitalizing just the first letter typically works better with keywords
|
||||||
|
that are like sentences (e.g. `Administrator logs into system`). These
|
||||||
|
type of keywords are often higher level.
|
||||||
|
|
||||||
|
Good:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
Login With Valid Credentials
|
||||||
|
|
||||||
|
Bad:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
Input Valid Username And Valid Password And Click Login Button
|
||||||
|
|
||||||
|
|
||||||
|
Naming setup and teardown
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
- Try to use name that describes what is done.
|
||||||
|
|
||||||
|
- Possibly use an existing keyword.
|
||||||
|
|
||||||
|
- More abstract names are acceptable if a setup or teardown contains unrelated
|
||||||
|
steps.
|
||||||
|
|
||||||
|
- Listing steps in name is duplication and a maintenance problem
|
||||||
|
(e.g. `Login to system, add user, activate alarms and check balance`).
|
||||||
|
|
||||||
|
- Often better to use something generic (e.g. `Initialize system`).
|
||||||
|
|
||||||
|
- BuiltIn keyword `Run Keywords`__ can work well if keywords implementing lower
|
||||||
|
level steps already exist.
|
||||||
|
|
||||||
|
- Not reusable so best used when the setup or teardown scenario is
|
||||||
|
needed only once.
|
||||||
|
|
||||||
|
- Everyone working with these tests should always understand what a setup or
|
||||||
|
teardown does.
|
||||||
|
|
||||||
|
Good:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
Suite Setup Initialize System
|
||||||
|
|
||||||
|
Good (if only used once):
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
Suite Setup Run Keywords
|
||||||
|
... Login To System AND
|
||||||
|
... Add User AND
|
||||||
|
... Activate Alarms AND
|
||||||
|
... Check Balance
|
||||||
|
|
||||||
|
Bad:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
Suite Setup Login To System, Add User, Activate Alarms And Check Balance
|
||||||
|
|
||||||
|
__ http://robotframework.org/robotframework/latest/libraries/
|
||||||
|
BuiltIn.html#Run%20Keywords
|
||||||
|
|
||||||
|
|
||||||
|
Documentation
|
||||||
|
=============
|
||||||
|
|
||||||
|
Test suite documentation
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
- Add overall documentation to test case files is mandatory.
|
||||||
|
|
||||||
|
- Should contain background information, why tests are created, notes about
|
||||||
|
execution environment, etc.
|
||||||
|
|
||||||
|
- Do not just repeat test suite name.
|
||||||
|
|
||||||
|
- Do not include too much details about test cases.
|
||||||
|
|
||||||
|
- Tests should be clear enough to understand alone.
|
||||||
|
- Duplicate information is waste and maintenance problem.
|
||||||
|
|
||||||
|
- Documentation can contain links to more information.
|
||||||
|
|
||||||
|
- Consider using `test suite metadata`__ if you need to document information
|
||||||
|
represented as name-value pairs (e.g. `Version: 1.0` or `OS: Linux`).
|
||||||
|
|
||||||
|
__ http://robotframework.org/robotframework/latest/
|
||||||
|
RobotFrameworkUserGuide.html#free-test-suite-metadata
|
||||||
|
|
||||||
|
- Documentation and metadata of the top level suite can be set from the
|
||||||
|
command line using `--doc` and `--metadata` options, respectively.
|
||||||
|
|
||||||
|
Good:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
Documentation Tests to verify that account withdrawals succeed and
|
||||||
|
... fail correctly depending from users account balance
|
||||||
|
... and account type dependent rules.
|
||||||
|
... See http://internal.example.com/docs/abs.pdf
|
||||||
|
Metadata Version 0.1
|
||||||
|
|
||||||
|
Bad (especially if suite is named well like `account_withdrawal.robot`):
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
Documentation Tests Account Withdrawal.
|
||||||
|
|
||||||
|
|
||||||
|
Test case documentation
|
||||||
|
-----------------------
|
||||||
|
|
||||||
|
- Test should have documentation. Only repeating the name of the test is not
|
||||||
|
valid.
|
||||||
|
|
||||||
|
- Test case structure should be clear enough without documentation or other
|
||||||
|
comments.
|
||||||
|
|
||||||
|
- Tags are generally more flexible and provide more functionality (statistics,
|
||||||
|
include/exclude, etc.) than documentation. Tags for type of configuration
|
||||||
|
are mandatory.
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
[Tags] Simplex Duplex MN-Local MN-External
|
||||||
|
|
||||||
|
Good:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Create Flavors for Cirros Instances
|
||||||
|
[Tags] Simplex Duplex MN-Local MN-External
|
||||||
|
[Documentation] Create flavors with or without properties to be used
|
||||||
|
... to launch Cirros instances.
|
||||||
|
${properties}= Catenate ${flavor_property_1} ${flavor_property_2}
|
||||||
|
Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk}
|
||||||
|
... ${flavor_name_1}
|
||||||
|
Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk}
|
||||||
|
... ${properties} ${flavor_name_2}
|
||||||
|
|
||||||
|
Bad:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Create Flavors for Cirros Instances
|
||||||
|
[Documentation] Create flavors giving a ram, vcpus and disk with or
|
||||||
|
... without properties o be used to launch Cirros instances on Simplex,
|
||||||
|
... Duplex or Multinode configuration for SaTarlingX deployment.
|
||||||
|
... All is done calling a keyword called Create Flavor with some
|
||||||
|
... parameters ... etc
|
||||||
|
[Tags] Simplex Duplex MN-Local MN-External
|
||||||
|
${properties}= Catenate ${flavor_property_1} ${flavor_property_2}
|
||||||
|
Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk}
|
||||||
|
... ${flavor_name_1}
|
||||||
|
Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk}
|
||||||
|
... ${properties} ${flavor_name_2}
|
||||||
|
|
||||||
|
|
||||||
|
User keyword documentation
|
||||||
|
--------------------------
|
||||||
|
|
||||||
|
- Its important document what keyword is doing.
|
||||||
|
|
||||||
|
- Important usage is documenting arguments and return values.
|
||||||
|
|
||||||
|
- Shown in resource file documentation generated with Libdoc__ and editors
|
||||||
|
such as RIDE__ can show it in keyword completion and elsewhere.
|
||||||
|
|
||||||
|
__ http://robotframework.org/robotframework/#built-in-tools
|
||||||
|
__ https://github.com/robotframework/RIDE
|
||||||
|
|
||||||
|
|
||||||
|
Test suite structure
|
||||||
|
====================
|
||||||
|
- All test should be placed under Test/X directory where X indicates the Domain
|
||||||
|
where Test Case belongs
|
||||||
|
|
||||||
|
- Tests in a suite should be related to each other.
|
||||||
|
|
||||||
|
- Common setup and/or teardown is often a good indicator.
|
||||||
|
|
||||||
|
- Should not have too many tests (max 10) in one file unless they are
|
||||||
|
`data-driven tests`_.
|
||||||
|
|
||||||
|
- Tests should be independent. Initialization using setup/teardown.
|
||||||
|
|
||||||
|
- Sometimes dependencies between tests cannot be avoided.
|
||||||
|
|
||||||
|
- For example, it can take too much time to initialize all tests separately.
|
||||||
|
- Never have long chains of dependent tests.
|
||||||
|
- Consider verifying the status of the previous test using the built-in
|
||||||
|
`${PREV TEST STATUS}` variable.
|
||||||
|
|
||||||
|
- We try to avoid the usage of keywords inside of a Test Suite unless they are
|
||||||
|
very specific to the test cases, if a keyword is designed and is generic,
|
||||||
|
this jeyword should be placed under Resources/ directory.
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
Run Command
|
||||||
|
[Arguments] ${cmd} ${fail_if_error}=False ${timeout}=${TIMEOUT}
|
||||||
|
[Documentation] Execute a command on controller over ssh connection
|
||||||
|
... keeping environment visible to the subsequent keywords.Also allows
|
||||||
|
... the keyword to fail if there is an error, by default this
|
||||||
|
... keyword will not fail and will return the stderr.
|
||||||
|
|
||||||
|
|
||||||
|
Test case structure
|
||||||
|
===================
|
||||||
|
|
||||||
|
- Test case should be easy to understand.
|
||||||
|
|
||||||
|
- Test case should be tagged and documented
|
||||||
|
|
||||||
|
- One test case should be testing one thing.
|
||||||
|
|
||||||
|
- *Things* can be small (part of a single feature) or large (end-to-end).
|
||||||
|
|
||||||
|
- Select suitable abstraction level.
|
||||||
|
|
||||||
|
- Use abstraction level consistently (single level of abstraction principle).
|
||||||
|
- Do not include unnecessary details on the test case level.
|
||||||
|
|
||||||
|
- We are using an strategy of `Workflow tests`_ for our development, please
|
||||||
|
follow the rules
|
||||||
|
|
||||||
|
- Try to not exceed 79 characters when possible, if it nos possible or the
|
||||||
|
test case get not esasy to read 99 chars are accepted.
|
||||||
|
|
||||||
|
Workflow tests
|
||||||
|
--------------
|
||||||
|
|
||||||
|
- Generally have these phases:
|
||||||
|
|
||||||
|
- Precondition (optional, often in setup)
|
||||||
|
- Action (do something to the system)
|
||||||
|
- Verification (validate results, mandatory)
|
||||||
|
- Cleanup (optional, always in teardown to make sure it is executed)
|
||||||
|
|
||||||
|
- Keywords describe what a test does.
|
||||||
|
|
||||||
|
- Use clear keyword names and suitable abstraction level.
|
||||||
|
- Should contain enough information to run manually.
|
||||||
|
- Should never need documentation or commenting to explain what the
|
||||||
|
test does.
|
||||||
|
|
||||||
|
- Different tests can have different abstraction levels.
|
||||||
|
|
||||||
|
- Tests for a detailed functionality are more precise.
|
||||||
|
- End-to-end tests can be on very high level.
|
||||||
|
- One test should use only one abstraction level
|
||||||
|
|
||||||
|
- Different styles:
|
||||||
|
|
||||||
|
- More technical tests for lower level details and integration tests.
|
||||||
|
- "Executable specifications" act as requirements.
|
||||||
|
- Use domain language.
|
||||||
|
- Everyone (including customer and/or product owner) should
|
||||||
|
always understand.
|
||||||
|
|
||||||
|
- Try to not use complex logic on the test case level.
|
||||||
|
|
||||||
|
- No for loops or if/else constructs as much as possible.
|
||||||
|
- Use variable assignments with care.
|
||||||
|
- Test cases should not look like scripts!
|
||||||
|
|
||||||
|
- Max 15 steps, preferably less.
|
||||||
|
|
||||||
|
Example using "normal" keyword-driven style:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Valid Login
|
||||||
|
Open Browser To Login Page
|
||||||
|
Input Username demo
|
||||||
|
Input Password mode
|
||||||
|
Submit Credentials
|
||||||
|
Welcome Page Should Be Open
|
||||||
|
|
||||||
|
User keywords
|
||||||
|
=============
|
||||||
|
|
||||||
|
- Should be easy to understand.
|
||||||
|
|
||||||
|
- Same rules as with workflow tests.
|
||||||
|
|
||||||
|
- Different abstraction levels.
|
||||||
|
|
||||||
|
- Can contain some programming logic (for loops, if/else).
|
||||||
|
|
||||||
|
- Especially on lower level keywords.
|
||||||
|
- Complex logic in libraries rather than in user keywords.
|
||||||
|
|
||||||
|
- Try to not exceed 79 characters when possible, if it nos possible or the
|
||||||
|
keyword becomes not esasy to read 99 chars are accepted.
|
||||||
|
|
||||||
|
|
||||||
|
Variables
|
||||||
|
=========
|
||||||
|
|
||||||
|
- Encapsulate long and/or complicated values.
|
||||||
|
|
||||||
|
- Pass information between keywords.
|
||||||
|
|
||||||
|
|
||||||
|
Variable naming
|
||||||
|
---------------
|
||||||
|
|
||||||
|
- Clear but not too long names.
|
||||||
|
|
||||||
|
- Use case consistently:
|
||||||
|
|
||||||
|
- Lower case with local variables only available inside a certain scope.
|
||||||
|
- Upper case with others (global, suite or test level).
|
||||||
|
- Both space and underscore can be used as a word separator.
|
||||||
|
|
||||||
|
- Recommended to also list variables that are set dynamically in the variable
|
||||||
|
table.
|
||||||
|
|
||||||
|
- Set typically using BuiltIn keyword `Set Suite Variable`__.
|
||||||
|
- The initial value should explain where/how the real value is set.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
Suite Setup Set Active User
|
||||||
|
|
||||||
|
*** Variables ***
|
||||||
|
# Default system address. Override when tested agains other instances.
|
||||||
|
${SERVER URL} http://sre-12.example.com/
|
||||||
|
${USER} Actual value set dynamically at suite setup
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
Set Active User
|
||||||
|
${USER} = Get Current User ${SERVER URL}
|
||||||
|
Set Suite Variable ${USER}
|
||||||
|
|
||||||
|
__ http://robotframework.org/robotframework/latest/libraries/
|
||||||
|
BuiltIn.html#Set%20Suite%20Variable
|
||||||
|
|
||||||
|
|
||||||
|
Passing and returning values
|
||||||
|
----------------------------
|
||||||
|
|
||||||
|
- Common approach is to return values from keywords, assign them to variables
|
||||||
|
and then pass them as arguments to other keywords.
|
||||||
|
|
||||||
|
- Clear and easy to follow approach.
|
||||||
|
- Allows creating independent keywords and facilitates re-use.
|
||||||
|
- Looks like programming and thus not so good on the test case level.
|
||||||
|
|
||||||
|
- Alternative approach is storing information in a library or using the BuiltIn
|
||||||
|
`Set Test Variable`__ keyword.
|
||||||
|
|
||||||
|
- Avoid programming style on the test case level as much as possible.
|
||||||
|
- Can be more complex to follow and make reusing keywords harder.
|
||||||
|
|
||||||
|
__ http://robotframework.org/robotframework/latest/libraries/
|
||||||
|
BuiltIn.html#Set%20Test%20Variable
|
||||||
|
|
||||||
|
Good for in suite keywords:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Withdraw From Account
|
||||||
|
Withdraw From Account $50
|
||||||
|
Withdraw Should Have Succeeded
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
Withdraw From Account
|
||||||
|
[Arguments] ${amount}
|
||||||
|
${STATUS} = Withdraw From User Account ${USER} ${amount}
|
||||||
|
Set Test Variable ${STATUS}
|
||||||
|
|
||||||
|
Withdraw Should Have Succeeded
|
||||||
|
Should Be Equal ${STATUS} SUCCESS
|
||||||
|
|
||||||
|
Good for generic keywords:
|
||||||
|
|
||||||
|
.. code:: robotframework
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
Withdraw From Account
|
||||||
|
${status} = Withdraw From Account $50
|
||||||
|
Withdraw Should Have Succeeded ${status}
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
Withdraw From Account
|
||||||
|
[Arguments] ${amount}
|
||||||
|
${status} = Withdraw From User Account ${USER} ${amount}
|
||||||
|
[Return] ${status}
|
||||||
|
|
||||||
|
Withdraw Should Have Succeeded
|
||||||
|
[Arguments] ${status}
|
||||||
|
Should Be Equal ${status} SUCCESS
|
||||||
|
|
||||||
|
|
||||||
|
Avoid sleeping
|
||||||
|
==============
|
||||||
|
|
||||||
|
- Sleeping is a very fragile way to synchronize tests.
|
||||||
|
|
||||||
|
- Safety margins cause too long sleeps on average.
|
||||||
|
|
||||||
|
- Instead of sleeps, use keyword that polls has a certain action occurred.
|
||||||
|
|
||||||
|
- Keyword names often starts with `Wait ...`.
|
||||||
|
- Should have a maximum time to wait.
|
||||||
|
- Possible to wrap other keywords inside the BuiltIn keyword
|
||||||
|
`Wait Until Keyword Succeeds`__.
|
||||||
|
|
||||||
|
- Sometimes sleeping is the easiest solution.
|
||||||
|
|
||||||
|
- Always use with care.
|
||||||
|
- Never use in user keywords that are used often by tests or other keywords.
|
||||||
|
|
||||||
|
- Can be useful in debugging to stop execution.
|
||||||
|
|
||||||
|
- `Dialogs library`__ often works better.
|
||||||
|
|
||||||
|
__ http://robotframework.org/robotframework/latest/libraries/
|
||||||
|
BuiltIn.html#Wait%20Until%20Keyword%20Succeeds
|
||||||
|
__ http://robotframework.org/robotframework/latest/libraries/Dialogs.html
|
|
@ -0,0 +1,251 @@
|
||||||
|
|
||||||
|
# Edge Compute Framework Structure documentation
|
||||||
|
|
||||||
|
Table of Contents
|
||||||
|
|
||||||
|
- [Introduction](#introduction)
|
||||||
|
- [Edge Compute Folder Structure](#edge-compute-folder-structure)
|
||||||
|
- [1. Libraries](#1-libraries)
|
||||||
|
- [2. Resources](#2-resources)
|
||||||
|
- [3. Variables](#3-variables)
|
||||||
|
- [4. Utils](#4-utils)
|
||||||
|
- [5. Tests](#5-test)
|
||||||
|
- [5.1 Horizon](#51-horizon)
|
||||||
|
- [5.2 Keystone](#52-keystone)
|
||||||
|
- [5.3 Nova](#53-nova)
|
||||||
|
- [5.4 Glance](#54-glance)
|
||||||
|
- [5.5 Swift](#55-swift)
|
||||||
|
- [5.6 Neutron](#56-cinder)
|
||||||
|
- [5.7 Cinder](#57-neutron)
|
||||||
|
- [5.8 Heat](#58-heat)
|
||||||
|
- [5.9 Ceilometer](#59-ceilometer)
|
||||||
|
- [6. Results](#6-results)
|
||||||
|
- [7. Latest-result](#7-latest-result)
|
||||||
|
|
||||||
|
|
||||||
|
## Introduction
|
||||||
|
This structure tries to fit Robot Framework User Guide good practices.<br>
|
||||||
|
For more detailed information please check the following :link:
|
||||||
|
[Link](http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#documentation-formatting)
|
||||||
|
|
||||||
|
## Edge Compute Folder Structure
|
||||||
|
|
||||||
|
```text
|
||||||
|
stx-test-suite/
|
||||||
|
|--Libraries
|
||||||
|
|--Resources
|
||||||
|
|--MyResources file
|
||||||
|
|--Variables
|
||||||
|
|--global.py file
|
||||||
|
|--Utils
|
||||||
|
|--Tests
|
||||||
|
|--Horizon component
|
||||||
|
|--Tests.robot file
|
||||||
|
|
||||||
|
*** Settings ***
|
||||||
|
[Documentation]
|
||||||
|
[Tags]
|
||||||
|
imports...
|
||||||
|
Resources
|
||||||
|
Test Setup
|
||||||
|
Test Teardown
|
||||||
|
|
||||||
|
*** Test Cases
|
||||||
|
...
|
||||||
|
|--__init__.robot file
|
||||||
|
|
||||||
|
***Settings***
|
||||||
|
imports
|
||||||
|
Suite Setup
|
||||||
|
Suite Teardown
|
||||||
|
|
||||||
|
***keywords***
|
||||||
|
...
|
||||||
|
|--Keystone
|
||||||
|
|--Nova
|
||||||
|
|--Glance
|
||||||
|
|--Swift
|
||||||
|
|--Neutron
|
||||||
|
|--Cinder
|
||||||
|
|--Heat
|
||||||
|
|--Ceilometer
|
||||||
|
|--Results
|
||||||
|
|--YYYYMMDDHHMMSS<test_suite_name>
|
||||||
|
|--Latest-results
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1. Libraries
|
||||||
|
Any test library that is **NOT** one of the standard libraries would be here.
|
||||||
|
|
||||||
|
### 2. Resources
|
||||||
|
Resource files contain user keywords used in test case files and test suite
|
||||||
|
initialization files providing a mechanism for sharing them.<br>
|
||||||
|
Since the resource file structure is very close to test case files,
|
||||||
|
it is easy to create them.
|
||||||
|
|
||||||
|
Resource files are imported using the _Resource setting_ in the settings table.
|
||||||
|
The path to the resource file is given in the cell after the setting name.
|
||||||
|
|
||||||
|
**e.g.**<br>
|
||||||
|
TEST CASE TEMPLATE <br>
|
||||||
|
```text
|
||||||
|
*** Settings ***
|
||||||
|
Resource my_resources.html
|
||||||
|
Resource ../data/resources.html
|
||||||
|
Resource ${RESOURCES}/common.tsv
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Variables
|
||||||
|
This folder should contain scripts declaring global variables, environmental
|
||||||
|
variables, or any special variable used in the Edge Computing framework.
|
||||||
|
|
||||||
|
**e.g.**<br>
|
||||||
|
|
||||||
|
- global.py
|
||||||
|
```text
|
||||||
|
CONTROLLER_IP='10.10.10.10'
|
||||||
|
CONTROLLER_USERNAME='root'
|
||||||
|
CONTROLLER_PASSWORD='linux'
|
||||||
|
```
|
||||||
|
|
||||||
|
- virtual_machine_variables.py
|
||||||
|
```text
|
||||||
|
- MEMDisk1=200GB
|
||||||
|
- MEMDisk2=100GB
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Utils
|
||||||
|
This folder should contain generic functions that perform actions across the
|
||||||
|
Edge Computing execution.
|
||||||
|
|
||||||
|
**e.g.**
|
||||||
|
|
||||||
|
- common_reports.py
|
||||||
|
```text
|
||||||
|
- check_results_dir(...)
|
||||||
|
- create_output_dir(...)
|
||||||
|
- link_latest_run(...)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Test
|
||||||
|
Contains the Robot Automated Test scenarios per Edge Computing Component.
|
||||||
|
|
||||||
|
#### 5.1 Horizon
|
||||||
|
Horizon folder should contain test cases focused on exercise the Horizon GUI
|
||||||
|
where every tenant or application user can launch up/manage their own instances.
|
||||||
|
|
||||||
|
**COMPONENT FOLDER STRUCTURE**
|
||||||
|
|
||||||
|
#### Tests.robot file
|
||||||
|
```text
|
||||||
|
*** Settings ***
|
||||||
|
[Documentation] tag specifying brief description of the Test case.
|
||||||
|
[Tags] keywords specifying what is being tested.
|
||||||
|
|
||||||
|
Imports...
|
||||||
|
# Import Order Template
|
||||||
|
|
||||||
|
# {{stdlib imports in human alphabetical order}}
|
||||||
|
# {{third-party lib imports in human alphabetical order}}
|
||||||
|
# {{project imports in human alphabetical order}}
|
||||||
|
|
||||||
|
# Import Order Example
|
||||||
|
import httplib
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
|
||||||
|
import eventlet
|
||||||
|
import webob.exc
|
||||||
|
|
||||||
|
import nova.api.ec2
|
||||||
|
from nova.api import manager
|
||||||
|
from nova.api import openstack
|
||||||
|
|
||||||
|
Resources...
|
||||||
|
Test Setup
|
||||||
|
Test Teardown
|
||||||
|
|
||||||
|
*** Test Cases ***
|
||||||
|
...
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### __init__.robot file
|
||||||
|
The **__ init__ file** is a special test suite initialization file.<br>
|
||||||
|
It is a test suite **created from a directory** that can have the same
|
||||||
|
structure and syntax as test case files, except that **they cannot have test
|
||||||
|
case tables** and **not** all settings are supported.
|
||||||
|
|
||||||
|
```text
|
||||||
|
*** Settings ***
|
||||||
|
Imports ...
|
||||||
|
Suite Setup
|
||||||
|
Suite Teardown
|
||||||
|
|
||||||
|
*** Keywords ***
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 5.2 Keystone
|
||||||
|
Keystone folder should contain test cases focused on exercise the authorization
|
||||||
|
or authentication related to every layer of services in OpenStack.<br>
|
||||||
|
|
||||||
|
**e.g.** Tokens, Catalog, Policy, and Assignment Service role.
|
||||||
|
|
||||||
|
|
||||||
|
#### 5.3 Nova
|
||||||
|
Nova folder should contain test cases focused on exercise compute domain.
|
||||||
|
|
||||||
|
**e.g.** When you launch a instance is where all the computing and all the
|
||||||
|
processing actually happens.
|
||||||
|
|
||||||
|
#### 5.4 Glance
|
||||||
|
Glance folder should contain test cases focused on exercise services for
|
||||||
|
launching the instances. Glance has all the disk images contained in them
|
||||||
|
different versions of OS such as Linux, CentOS, Ubuntu, Windows.
|
||||||
|
|
||||||
|
|
||||||
|
#### 5.5 Swift
|
||||||
|
Swift folder should contain test cases focused on exercise databases as a
|
||||||
|
Object Storage Services where you can store all kind of files `as objects`.
|
||||||
|
|
||||||
|
**REMARK:** Everything is converted to an object and saved in the Storage File
|
||||||
|
System.
|
||||||
|
|
||||||
|
|
||||||
|
#### 5.6 Cinder
|
||||||
|
Cinder folder should contain test cases focused on exercise databases as a
|
||||||
|
`Block storage` services where the database is seen like a pluggable storage
|
||||||
|
which is very similar to the disk storage system in a computer.
|
||||||
|
|
||||||
|
|
||||||
|
#### 5.7 Neutron
|
||||||
|
Neutron folder should contain test cases focused on exercise the `networking`
|
||||||
|
service associated with all OpenStack Services.
|
||||||
|
|
||||||
|
#### 5.8 Heat
|
||||||
|
Heat folder should contain test cases focused on exercise Orchestration.
|
||||||
|
|
||||||
|
#### 5.9 Ceilometer
|
||||||
|
Ceilometer folder should contain tests cases focused on exercise metering and
|
||||||
|
billing. It checks availability of the services for how much time is an instance
|
||||||
|
active or running.
|
||||||
|
|
||||||
|
### 6. Results
|
||||||
|
Test execution results are keep in folders following the next format when the
|
||||||
|
execution was ran:
|
||||||
|
```text
|
||||||
|
%Y%m%d%H%M%S_<test_suite_name>
|
||||||
|
Where
|
||||||
|
%Y -- Year
|
||||||
|
%m -- Month
|
||||||
|
%d -- Day
|
||||||
|
%H%M%S -- Hour Minute Second
|
||||||
|
```
|
||||||
|
|
||||||
|
### 7. Latest-result
|
||||||
|
<br>...
|
||||||
|
|
||||||
|
Hope this can help you good look :+1:
|
|
@ -0,0 +1,7 @@
|
||||||
|
# configuration for flake8
|
||||||
|
[flake8]
|
||||||
|
ignore = W191
|
||||||
|
exclude = .venv,.tox,dist,doc,build,*.egg
|
||||||
|
import-order-style = pep8
|
||||||
|
max-line-length = 79
|
||||||
|
max-complexity = 15
|
|
@ -0,0 +1,53 @@
|
||||||
|
# tox configuration
|
||||||
|
[tox]
|
||||||
|
skipsdist = True
|
||||||
|
envlist = flake8, py27, py36
|
||||||
|
|
||||||
|
[testenv]
|
||||||
|
setenv = VIRTUAL_ENV={envdir}
|
||||||
|
PYTHONPATH={toxinidir}
|
||||||
|
# passed to 'pip install --prie', that will install the dependencies
|
||||||
|
# listed in those files
|
||||||
|
deps = -r{toxinidir}/test-requirements.txt
|
||||||
|
|
||||||
|
# => Linters
|
||||||
|
# ==========
|
||||||
|
|
||||||
|
# settings specific to the flake8 environment
|
||||||
|
[testenv:flake8]
|
||||||
|
basepython = python3
|
||||||
|
skip_install = True
|
||||||
|
# The command to run:
|
||||||
|
commands = flake8 --statistics --count --hang-closing --max-line-length=79 --show-source --import-order-style=pep8 {posargs}
|
||||||
|
# we only need flake8 and hacking when linting,
|
||||||
|
|
||||||
|
[testenv:venv]
|
||||||
|
# let you pass additional arguments when invoking tox
|
||||||
|
commands = {posargs}
|
||||||
|
|
||||||
|
[testenv:py27]
|
||||||
|
commands = python -m unittest {posargs:discover -vs .}
|
||||||
|
|
||||||
|
[testenv:coverage]
|
||||||
|
commands = coverage erase
|
||||||
|
coverage run --source=stx-test-suite -m unittest {posargs:discover -vs .}
|
||||||
|
coverage html
|
||||||
|
coverage report --fail-under=80
|
||||||
|
|
||||||
|
[testenv:pylint]
|
||||||
|
commands = pylint {posargs}
|
||||||
|
deps = -r{toxinidir}/test-requirements.txt
|
||||||
|
#-r{toxinidir}/requirements.txt
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
exclude = .git,__pycache__,old,build,dist
|
||||||
|
max-complexity = 15
|
||||||
|
count = True
|
||||||
|
statistics = True
|
||||||
|
hang-closing = True
|
||||||
|
max-line-length = 79
|
||||||
|
show-source = True
|
||||||
|
import-order-style = pep8
|
||||||
|
verbose = 1
|
||||||
|
jobs = 2
|
||||||
|
show-pep8 = True
|
Loading…
Reference in New Issue