pytest automation apparently running tests during test collection phase

Question

Running webui automated tests with pytest and selenium; having an issue where it appears that my tests are actually running during the collection phase. During this phase, I would expect pytest to be collecting tests - not running them. The end result is I end up with 6 test results where I would expect 2. Now the interesting piece, the 6 results only appear in the HTML report; on the command line I only get the expected 2 lines of output (but it 300 seconds to run those two tests because the tests are literally running multiple times).

tests/test_datadriven.py

#!/usr/bin/env python
from unittestzero import Assert
from pages.home import Home
from pages.administration import RolesTab
from api.api import ApiTasks
import time
import pytest

from data.datadrv import *

class TestRolesDataDriven(object):
    scenarios = [scenario1,scenario2]

    @pytest.mark.challenge
    def test_datadriven_rbac(self, mozwebqa, org, perm_name, resource, verbs, allowed, disallowed):
        """
Perform a data driven test related to role based access controls.
All parameters are fullfilled by the data.
:param org: Organization Name
:param perm_name: Permission name
:param resource: Resource
:param verbs: A tuple of verbs
:returns: Pass or Fail for the test
"""

        sysapi = ApiTasks(mozwebqa)
        home_page = Home(mozwebqa)
        rolestab = RolesTab(mozwebqa)

        role_name = "role_%s" % (home_page.random_string())
        perm_name = "perm_%s" % (home_page.random_string())
        username = "user%s" % home_page.random_string()
        email = username + "@example.com"
        password = "redhat%s" % (home_page.random_string())

        sysapi.create_org(org)
        sysapi.create_user(username, password, email)

        home_page.login()

        home_page.tabs.click_tab("administration_tab")
        home_page.tabs.click_tab("roles_administration")
        home_page.click_new()
        rolestab.create_new_role(role_name)

        rolestab.click_role_permissions()

        rolestab.role_org(org).click()
        rolestab.click_add_permission()

        rolestab.select_resource_type(resource)
        home_page.click_next()
        for v in verbs:
            home_page.select('verbs', v)
        home_page.click_next()

        rolestab.enter_permission_name(perm_name)
        rolestab.enter_permission_desc('Added by QE test.')
        rolestab.click_permission_done()

        rolestab.click_root_roles()
        rolestab.click_role_users()

        rolestab.role_user(username).add_user()

        home_page.header.click_logout()
        home_page.login(username, password)

        for t in allowed:
            Assert.true(t(home_page))

        for t in disallowed:
            Assert.false(t(home_page))

data/data.py

###
# DO NOT EDIT HERE
###

def pytest_generate_tests(metafunc):
    """
Parse the data provided in scenarios.
"""
    idlist = []
    argvalues = []
    for scenario in metafunc.cls.scenarios:
        idlist.append(scenario[0])
        items = scenario[1].items()
        argnames = [x[0] for x in items]
        argvalues.append(([x[1] for x in items]))
    metafunc.parametrize(argnames, argvalues, ids=idlist)
###
# EDIT BELOW
# ADD NEW SCENARIOS
###

scenario1 = ('ACME_Manage_Keys', { 'org': 'ACME_Corporation',
                                   'perm_name': 'ManageAcmeCorp',
                                   'resource': 'activation_keys',
                                   'verbs': ('manage_all',),
                                   'allowed': (Base.is_system_tab_visible,
                                               Base.is_new_key_visible,
                                               Base.is_activation_key_name_editable),
                                   'disallowed': (Base.is_dashboard_subs_visible,)})
scenario2 = ('Global_Read_Only', { 'org': 'Global Permissions',
                                   'perm_name': 'ReadOnlyGlobal',
                                   'resource': 'organizations',
                                   'verbs': ('read','create'),
                                   'allowed': (Base.is_organizations_tab_visible,
                                               Base.is_new_organization_visible,
                                               Base.is_new_organization_name_field_editable),
                                   'disallowed': (Base.is_system_tab_visible,
                                                  Base.is_new_key_visible)})

Full source is available at github; https://github.com/eanxgeek/katello_challenge

Anyone have any idea what might be going on here? I am using the pytest-mozwebqa plugin, pytests, and selenium.

Thanks!

Answer 1

Check the version of pytest-mozwebqa you have installed. If your installed version is < 0.10 then you must update.

pip-python install --upgrade pytest-mozwebqa

Due to the number of changes in pytest-mozwebqa I strongly encourage you to test first in a python virtualenv.