[checkmk-commits] Cleaned up test targets in Makefile

Module: check_mk Branch: master Commit: 0a9101410ab9c0a5618ea71b2c7c4807ba1693ed URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=0a9101410ab9c0… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Mon Mar 26 09:45:54 2018 +0200 Cleaned up test targets in Makefile * Use the new mandatory "-T" argument of py.test to call the tests of a given type. * Add "test-unit" target to execute all unit tests Change-Id: I9081514eef71778788b3133e5ca64313ee3a13b4 --- pytest.ini | 5 +- tests/Makefile | 22 ++++--- tests/conftest.py | 96 ++++++++++++++++++++++++++++--- tests/git/test_find_debug_code_web.py | 3 - tests/git/test_find_debug_print.py | 4 -- tests/git/test_permissions.py | 4 -- tests/packaging/test_files.py | 5 +- tests/pylint/test_pylint_check_plugins.py | 4 -- tests/pylint/test_pylint_misc.py | 4 -- tests/pylint/test_pylint_web.py | 4 -- tests/unit/conftest.py | 2 - 11 files changed, 104 insertions(+), 49 deletions(-) diff --git a/pytest.ini b/pytest.ini index 43f5b97..88b64f4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,11 +4,8 @@ testpaths = tests # integration tests work which are executed as site user. addopts = -p no:stepwise -p no:cacheprovider -rs -v --maxfail=1 # Register some markers to classify the tests +# TODO: Are they used? Clean them up markers = - pylint: Mark a test as pylint test. - git: Mark a test to be executed in the Git repository context - packaging: This is a test to execute to verify the packaging of the testbuilds. - gui_crawl: All linked pages of the Web interface are visited and searched for errors html_gentest: Generate test cases for htmllib refactoring. checks: Run all existing test cases for checks. filterwarnings = diff --git a/tests/Makefile b/tests/Makefile index e129d49..0f8ab66 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,31 +1,37 @@ SHELL=/bin/bash -e +PYTEST=py.test -s -vv .PHONY: help test-pylint test-integration test-gui-crawl \ test-bandit test-shellcheck setup format test-mypy \ - test-pylint-simple + test-pylint-simple test-unit test-packaging help: @echo "setup - Install dependencies" - @echo "test-integration - Run integration tests" + @echo "test-unit - Run unit tests" @echo "test-pylint - Run pylint based tests" + @echo "test-integration - Run integration tests" @echo "test-gui-crawl - Run GUI crawl test" @echo "test-bandit - Run bandit (security) tests" @echo "test-shellcheck - Run shellcheck tests" +test-unit: + $(PYTEST) -T unit + test-integration: - py.test \ - -m "not pylint and not packaging and not git and not html_gentest and not gui_crawl" \ - -s \ - -vv + $(PYTEST) -T integration test-gui-crawl: - py.test -s -m gui_crawl + $(PYTEST) -T gui_crawl test-pylint: - py.test -m pylint + $(PYTEST) -T pylint + + +test-packageing: + $(PYTEST) -T packaging test-bandit: diff --git a/tests/conftest.py b/tests/conftest.py index fa956d2..299e61b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,79 @@ import testlib import tempfile import shutil import errno +from collections import OrderedDict + +# +# Each test is of one of the following types. +# +# The tests are marked using the marker pytest.marker.type("TYPE") +# which is added to the test automatically according to their location. +# +# With each call to py.test one type of tests needs to be selected using +# the "-T TYPE" option. Only these tests will then be executed. Tests of +# the other type will be skipped. +# + +EXECUTE_IN_SITE, EXECUTE_IN_VENV = True, False + +test_types = OrderedDict([ + ("unit", EXECUTE_IN_VENV), + ("pylint", EXECUTE_IN_VENV), + ("integration", EXECUTE_IN_SITE), + ("gui_crawl", EXECUTE_IN_SITE), + ("packaging", EXECUTE_IN_VENV), +]) + +def pytest_addoption(parser): + """Register the -T option to pytest""" + parser.addoption("-T", action="store", metavar="TYPE", default=None, + help="Run tests of the given TYPE. Available types are: %s" % + ", ".join(test_types.keys())) + +def pytest_configure(config): + """Register the type marker to pytest""" + config.addinivalue_line("markers", + "type(TYPE): Mark TYPE of test. Available: %s" % + ", ".join(test_types.keys())) + + +def pytest_collection_modifyitems(items): + """Mark collected test types based on their location""" + for item in items: + type_marker = item.get_marker("type") + if type_marker and type_marker.args: + continue # Do not modify manually set marks + + file_path = "%s" % item.reportinfo()[0] + if "tests/unit" in file_path: + ty = "unit" + elif "tests/git" in file_path: + ty = "unit" + elif "tests/packaging" in file_path: + ty = "packaging" + elif "tests/pylint" in file_path: + ty = "pylint" + elif "tests/integration" in file_path: + ty = "integration" + else: + raise Exception("Test not TYPE marked: %r" % item) + + item.add_marker(pytest.mark.type.with_args(ty)) + + +def pytest_runtest_setup(item): + """Skip tests of unwanted types""" + test_type = item.get_marker("type") + if test_type is None: + raise Exception("Test is not TYPE marked: %s" % item) + + if not item.config.getoption("-T"): + raise SystemExit("Please specify type of tests to be executed (py.test -T TYPE)") + + test_type_name = test_type.args[0] + if test_type_name != item.config.getoption("-T"): + pytest.skip("Not testing type %r" % test_type_name) + # Some cmk.* code is calling things like cmk. is_raw_edition() at import time # (e.g. cmk_base/default_config/notify.py) for edition specific variable @@ -75,14 +148,21 @@ def add_python_paths(): def pytest_cmdline_main(config): - # Some special tests are not executed in a site environment, but in - # virtualenv environment. The integration tests and so on are required to - # run in our runtime environment (the site). Things like unit tests just - # need a similar python environment having the required modules. - verify_virtualenv() - #if config.getoption('markexpr') in [ "packaging", "git", "html_gentest", "unit" ]: - #else: - # setup_site_and_switch_user() + """There are 2 environments for testing: + + * A real Check_MK site environment (e.g. integration tests) + * Python virtual environment (e.g. for unit tests) + + Depending on the selected "type" marker the environment is ensured + or switched here.""" + if not config.getoption("-T"): + return # missing option is handled later + + context = test_types[config.getoption("-T")] + if context == EXECUTE_IN_SITE: + setup_site_and_switch_user() + else: + verify_virtualenv() def verify_virtualenv(): diff --git a/tests/git/test_find_debug_code_web.py b/tests/git/test_find_debug_code_web.py index 4397efe..82f4fd6 100644 --- a/tests/git/test_find_debug_code_web.py +++ b/tests/git/test_find_debug_code_web.py @@ -6,9 +6,6 @@ import glob import pytest from testlib import cmk_path, cmc_path -# Mark all tests in this file to be executed in the git context -pytestmark = pytest.mark.git - def test_find_debug_code(): scanned = 0 for base_path in [ cmk_path(), cmc_path() ]: diff --git a/tests/git/test_find_debug_print.py b/tests/git/test_find_debug_print.py index b08ddc2..1c8b824 100644 --- a/tests/git/test_find_debug_print.py +++ b/tests/git/test_find_debug_print.py @@ -3,12 +3,8 @@ import os import glob -import pytest from testlib import cmk_path, cmc_path, cme_path -# Mark all tests in this file to be executed in the git context -pytestmark = pytest.mark.git - check_paths = [ "bin", "cmk_base", diff --git a/tests/git/test_permissions.py b/tests/git/test_permissions.py index a5a3bff..db43a85 100644 --- a/tests/git/test_permissions.py +++ b/tests/git/test_permissions.py @@ -3,12 +3,8 @@ import os import glob -import pytest from testlib import cmk_path -# Mark all tests in this file to be executed in the git context -pytestmark = pytest.mark.git - def is_executable(path): return os.path.isfile(path) and os.access(path, os.X_OK) diff --git a/tests/packaging/test_files.py b/tests/packaging/test_files.py index 955fd44..7a666e4 100644 --- a/tests/packaging/test_files.py +++ b/tests/packaging/test_files.py @@ -1,10 +1,7 @@ -import pytest import os import subprocess import re - -# Mark all tests in this file to be tests verifying build packages -pytestmark = pytest.mark.packaging +import pytest @pytest.mark.parametrize("what", [ ("rpm"), diff --git a/tests/pylint/test_pylint_check_plugins.py b/tests/pylint/test_pylint_check_plugins.py index c2efcd5..a259872 100755 --- a/tests/pylint/test_pylint_check_plugins.py +++ b/tests/pylint/test_pylint_check_plugins.py @@ -3,14 +3,10 @@ import os import sys -import pytest from testlib import repo_path, cmc_path import testlib.pylint_cmk as pylint_cmk -# Mark all tests in this file to be pylint checks -pytestmark = pytest.mark.pylint - def test_pylint_checks(pylint_test_dir): f = file(pylint_test_dir + "/cmk-checks.py", "w") diff --git a/tests/pylint/test_pylint_misc.py b/tests/pylint/test_pylint_misc.py index 4430452..b72fdb1 100755 --- a/tests/pylint/test_pylint_misc.py +++ b/tests/pylint/test_pylint_misc.py @@ -3,14 +3,10 @@ import os import sys -import pytest from testlib import cmk_path, cmc_path, cme_path import testlib.pylint_cmk as pylint_cmk -# Mark all tests in this file to be pylint checks -pytestmark = pytest.mark.pylint - def test_pylint_misc(): search_paths = [ cmk_path() + "/cmk_base", diff --git a/tests/pylint/test_pylint_web.py b/tests/pylint/test_pylint_web.py index 6a0749d..50af9e3 100755 --- a/tests/pylint/test_pylint_web.py +++ b/tests/pylint/test_pylint_web.py @@ -5,14 +5,10 @@ import os import sys import glob import tempfile -import pytest from testlib import cmk_path, cmc_path, cme_path import testlib.pylint_cmk as pylint_cmk -# Mark all tests in this file to be pylint checks -pytestmark = pytest.mark.pylint - def get_web_plugin_dirs(): plugin_dirs = sorted(list(set(os.listdir(cmk_path() + "/web/plugins") + os.listdir(cmc_path() + "/web/plugins") diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 30ebce6..a896b08 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,7 +1,5 @@ import pytest -pytestmark = pytest.mark.unit - # Unit tests should not be executed in site. # -> Disabled site fixture for them @pytest.fixture(scope="session")