[checkmk-commits] Add make targets for YAPF formatting and CI integration

Module: check_mk Branch: master Commit: d5c06b1d6c9a3104186b35abe2824de12d8f857a URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=d5c06b1d6c9a31… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Mon Nov 12 16:31:20 2018 +0100 Add make targets for YAPF formatting and CI integration All use YAPF from our standard virtual environment to format the source files. * Add make target "format-python" to top level file This target uses the YAPF from our virtual environment to format all our python source files. * Add make target "test-format-python" to tests/ This target can be used to tests formatting of all python files. It is used by our CI to warn about unformatted code. In case we have a specific set of files to check, the environment variable PYTHON_FILES can be set to check the listed files instead of all known python files. * Excluded tests directory from "find-python-files" output because the data structures in some tests dramatically increase the execution time of YAPF (8 minutes -> 60 minutes) CMK-1179 Change-Id: I9cf82c3e5360c5bcf81e717431750d26f0d304c1 --- Makefile | 15 +++++++++++++-- tests/Makefile | 14 +++++++++++++- tests/find-python-files | 6 ++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bebb292..c0f61e1 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ PNG_FILES := $(wildcard $(addsuffix /*.png,web/htdocs/images web/htdocs .PHONY: all analyze build check check-binaries check-permissions \ - check-version clean compile-neb-cmc cppcheck dist documentation format \ + check-version clean compile-neb-cmc cppcheck dist documentation format format-c format-python \ GTAGS headers help install iwyu mrproper \ optimize-images packages setup setversion tidy version \ am--refresh skel @@ -477,12 +477,23 @@ ifeq ($(ENTERPRISE),yes) $(MAKE) -C enterprise/core/src cppcheck-xml endif +format: format-python format-c + # TODO: We should probably handle this rule via AM_EXTRA_RECURSIVE_TARGETS in # src/configure.ac, but this needs at least automake-1.13, which in turn is only # available from e.g. Ubuntu Saucy (13) onwards, so some magic is needed. -format: +format-c: $(CLANG_FORMAT) -style=file -i $(FILES_TO_FORMAT) +format-python: .venv +# Explicitly specify --style [FILE] to prevent costly searching in parent directories +# for each file specified via command line +# +# Saw some mixed up lines on stdout after adding the --parallel option. Leaving it on +# for the moment to get the performance boost this option brings. + PYTHON_FILES=$${PYTHON_FILES-$$(tests/find-python-files)} ; \ + $(PIPENV) run yapf --parallel --style .style.yapf --verbose -i $$PYTHON_FILES + # Note: You need the doxygen and graphviz packages. documentation: config.h $(MAKE) -C livestatus/src documentation diff --git a/tests/Makefile b/tests/Makefile index 0ba7f03..96cf581 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -19,11 +19,12 @@ help: @echo "test-bandit - Run bandit (security) tests" @echo "test-shellcheck - Run shellcheck tests" @echo "test-docker - Run docker tests" + @echo "test-format-python - Test the python formatting" # This target needs to be phony so it is run every time because only the other # makefile can determine that there's nothing to be done. # TODO: Move everything to top level makefile? -.PHONY: ../.venv bandit.ini +.PHONY: ../.venv bandit.ini test-format-python ../.venv: $(MAKE) -C .. .venv @@ -31,6 +32,17 @@ help: pipenv-check: ../.venv $(PIPENV) check +test-format-python: ../.venv +# Explicitly specify --style [FILE] to prevent costly searching in parent directories +# for each file specified via command line +# +# Saw some mixed up lines on stdout after adding the --parallel option. Leaving it on +# for the moment to get the performance boost this option brings. + PYTHON_FILES=$${PYTHON_FILES-$$(./find-python-files)} ; \ + $(PIPENV) run yapf --parallel --style ../.style.yapf --verbose --diff $$PYTHON_FILES \ + | grep -e "^Reformatting " -e "(reformatted)$$" \ + | awk '{if ($$1 == "+++") { print "Unformatted: "$$2 } else {print "Checking "$$2"..."}}' ; exit $${PIPESTATUS[0]} + test-unit: ../.venv $(PYTEST) -T unit diff --git a/tests/find-python-files b/tests/find-python-files index 19d0056..d846fb8 100755 --- a/tests/find-python-files +++ b/tests/find-python-files @@ -22,9 +22,11 @@ SEARCH+=" notifications" SEARCH+=" omd/packages/cma" SEARCH+=" omd/packages/maintenance" SEARCH+=" omd/packages/omd" - SEARCH+=" scripts" -SEARCH+=" tests" +# Tests contain some very large data structures (e.g. +# tests/unit/checks/test_docker_parse_node_images.py). Skip them for the +# moment because they make YAPF run more than 60 minutes instead of 8 minutes +#SEARCH+=" tests" SEARCH+=" web/app" SEARCH+=" werk"