[checkmk-commits] cmk-update-agent: Support build of cmk-update-agent within chroot-jail

Module: check_mk Branch: master Commit: f2b93112c136c774a723e782cd1577cfe65fa8f6 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=f2b93112c136c7… Author: Andreas Umbreit <au(a)mathias-kettner.de> Date: Fri Nov 3 09:16:25 2017 +0100 cmk-update-agent: Support build of cmk-update-agent within chroot-jail - Use chroot jail from windows agent build - Add needed depencencies to windows agent make-chroot script - Make windows agent chroot scripts independend of current working directory - Update windows agent updater makefiles to support a parallel and incremental build - Update .gitignore files for a clean build on buildserver Change-Id: I7a7110e3427446288288ce158f2373792269f2c7 --- agents/windows/chroot_version | 2 +- agents/windows/do-chroot | 22 +++- .../windows/frozen_binaries/pyinstaller-deps.make | 122 +++++++++++++-------- agents/windows/make-chroot | 9 +- 4 files changed, 97 insertions(+), 58 deletions(-) diff --git a/agents/windows/chroot_version b/agents/windows/chroot_version index 1e8b314..7f8f011 100644 --- a/agents/windows/chroot_version +++ b/agents/windows/chroot_version @@ -1 +1 @@ -6 +7 diff --git a/agents/windows/do-chroot b/agents/windows/do-chroot index 35ac818..5d7134c 100755 --- a/agents/windows/do-chroot +++ b/agents/windows/do-chroot @@ -7,7 +7,8 @@ CMD="$1" export HOSTNAME=cmk-windows-agent-chroot unset LANG -BASE="$(realpath ../..)" +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +BASE="$(realpath $SCRIPT_DIR/../..)" if [ -z $CHROOT_PATH ]; then echo "ERROR: CHROOT_PATH not set" @@ -48,11 +49,20 @@ else EXIT_CODE=$? fi -umount $CHROOT_PATH/$CHROOT_BUILD_DIR +function unmount() +{ + target=$1 + umount $target + if [ $? != 0 ] ; then + echo "retrying after 5 seconds..." + sleep 5 + umount $target && echo "successful" + fi +} -umount $CHROOT_PATH/dev/pts -sleep 4 -umount $CHROOT_PATH/dev -umount $CHROOT_PATH/proc +unmount $CHROOT_PATH/dev/pts +unmount $CHROOT_PATH/dev +unmount $CHROOT_PATH/proc +unmount $CHROOT_PATH/$CHROOT_BUILD_DIR exit $EXIT_CODE diff --git a/agents/windows/frozen_binaries/pyinstaller-deps.make b/agents/windows/frozen_binaries/pyinstaller-deps.make index fb9a930..9768246 100644 --- a/agents/windows/frozen_binaries/pyinstaller-deps.make +++ b/agents/windows/frozen_binaries/pyinstaller-deps.make @@ -8,72 +8,98 @@ PLUGINS_DIR = $(shell realpath ./../../plugins) # This package list originates from resolving the dependencies of the packages # pyinstaller, pyopenssl, requests. The required packages are explicitly listed # in favor of providing a working setup for a pyinstaller build with python 2.7 + +asn1crypto = src/pip/asn1crypto-0.22.0-py2.py3-none-any.whl +certifi = src/pip/certifi-2017.7.27.1-py2.py3-none-any.whl +cffi = src/pip/cffi-1.10.0-cp27-cp27m-win32.whl +chardet = src/pip/chardet-3.0.4-py2.py3-none-any.whl +crytography = src/pip/cryptography-2.0.3-cp27-cp27m-win32.whl +enum34 = src/pip/enum34-1.1.6-py2-none-any.whl +future = src/pip/future-0.16.0.tar.gz +idna = src/pip/idna-2.6-py2.py3-none-any.whl +ipaddress = src/pip/ipaddress-1.0.18-py2-none-any.whl +pycparser = src/pip/pycparser-2.18.tar.gz +PyInstaller = src/pip/PyInstaller-3.2.1.tar.bz2 +pyOpenSSL = src/pip/pyOpenSSL-17.2.0-py2.py3-none-any.whl +pypiwin32 = src/pip/pypiwin32-219-cp27-none-win32.whl +requests = src/pip/requests-2.18.4-py2.py3-none-any.whl +setuptools = src/pip/setuptools-36.2.7-py2.py3-none-any.whl +six = src/pip/six-1.10.0-py2.py3-none-any.whl +urllib3 = src/pip/urllib3-1.22-py2.py3-none-any.whl + PYTHON_PACKAGES = \ -src/pip/asn1crypto-0.22.0-py2.py3-none-any.whl \ -src/pip/certifi-2017.7.27.1-py2.py3-none-any.whl \ -src/pip/cffi-1.10.0-cp27-cp27m-win32.whl \ -src/pip/chardet-3.0.4-py2.py3-none-any.whl \ -src/pip/cryptography-2.0.3-cp27-cp27m-win32.whl \ -src/pip/enum34-1.1.6-py2-none-any.whl \ -src/pip/future-0.16.0.tar.gz \ -src/pip/idna-2.6-py2.py3-none-any.whl \ -src/pip/ipaddress-1.0.18-py2-none-any.whl \ -src/pip/pycparser-2.18.tar.gz \ -src/pip/PyInstaller-3.2.1.tar.bz2 \ -src/pip/pyOpenSSL-17.2.0-py2.py3-none-any.whl \ -src/pip/pypiwin32-219-cp27-none-win32.whl \ -src/pip/requests-2.18.4-py2.py3-none-any.whl \ -src/pip/setuptools-36.2.7-py2.py3-none-any.whl \ -src/pip/six-1.10.0-py2.py3-none-any.whl \ -src/pip/urllib3-1.22-py2.py3-none-any.whl + $(asn1crypto) \ + $(certifi) \ + $(cffi) \ + $(chardet) \ + $(crytography) \ + $(enum34) \ + $(future) \ + $(idna) \ + $(ipaddress) \ + $(pycparser) \ + $(PyInstaller) \ + $(pyOpenSSL) \ + $(pypiwin32) \ + $(requests) \ + $(setuptools) \ + $(six) \ + $(urllib3) + +$(asn1crypto) := asn1crypto==0.22.0 +$(certifi) := certifi==2017.7.27.1 +$(cffi) := cffi==1.10.0 +$(chardet) := chardet==3.0.4 +$(crytography) := cryptography==2.0.3 +$(enum34) := enum34==1.1.6 +$(future) := future==0.16.0 +$(idna) := idna==2.6 +$(ipaddress) := ipaddress==1.0.18 +$(pycparser) := pycparser==2.18 +$(PyInstaller) := PyInstaller==3.2.1 +$(pyOpenSSL) := pyOpenSSL==17.2.0 +$(pypiwin32) := pypiwin32==219 +$(requests) := requests==2.18.4 +$(setuptools) := setuptools==36.2.7 +$(six) := six==1.10.0 +$(urllib3) := urllib3==1.22 src/python-$(PYTHON_VERSION).msi: - cd src && wget https://www.python.org/ftp/python/$(PYTHON_VERSION)/python-$(PYTHON_VERSION… + mkdir -p src && \ + cd src && \ + curl -O https://www.python.org/ftp/python/$(PYTHON_VERSION)/python-$(PYTHON_VERSION… + +$(BUILD_DIR)/drive_c/python.exe: src/python-$(PYTHON_VERSION).msi + mkdir -p $(BUILD_DIR) && \ + export WINEPREFIX=$(BUILD_DIR) && \ + cd $(BUILD_DIR) && \ + cp $(CURDIR)/src/python-$(PYTHON_VERSION).msi . && \ + wine msiexec /qn /i python-$(PYTHON_VERSION).msi && \ + cd $(CURDIR) -$(PYTHON_PACKAGES): src/python-$(PYTHON_VERSION).msi +$(PYTHON_PACKAGES): $(BUILD_DIR)/drive_c/Python27/python.exe # Download needed python packages including depencencies. This has to be done # from within wine to obtain the correct win32 packages. # Note: We built this list to make the agent compilation reproducable. From time # to time we should update the packages here, but we don't want to fetch new versions # automatically. - mkdir $(BUILD_DIR) && \ + mkdir -p $(BUILD_DIR) && \ export WINEPREFIX=$(BUILD_DIR) && \ cd $(BUILD_DIR) && \ - cp $(CURDIR)/src/python-$(PYTHON_VERSION).msi . && \ - wine msiexec /qn /i python-$(PYTHON_VERSION).msi && \ - mkdir pip && \ + mkdir -p pip && \ cd pip && \ - wine c:\\Python27\\python.exe -m pip download --no-deps asn1crypto==0.22.0 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps certifi==2017.7.27.1 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps cffi==1.10.0 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps chardet==3.0.4 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps cryptography==2.0.3 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps enum34==1.1.6 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps future==0.16.0 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps idna==2.6 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps ipaddress==1.0.18 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps pycparser==2.18 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps PyInstaller==3.2.1 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps pyOpenSSL==17.2.0 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps pypiwin32==219 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps requests==2.18.4 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps setuptools==36.2.7 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps six==1.10.0 && \ - wine c:\\Python27\\python.exe -m pip download --no-deps urllib3==1.22 && \ + wine c:\\Python27\\python.exe -m pip download --no-deps $($@) && \ mkdir -p $(CURDIR)/src/pip && \ cp -r * $(CURDIR)/src/pip - rm -rf $(BUILD_DIR) -new_packages: src/python-$(PYTHON_VERSION).msi +new_packages: $(BUILD_DIR)/drive_c/Python27/python.exe # Use this target to obtain the newest versions of the needed packages. # You should update the explicit dependencies afterwars because the automatic # download of the latests packages might lead to an incosistent state, e.g. # if a package gets downloaded multiple times with different versions. - mkdir $(BUILD_DIR) && \ + mkdir -p $(BUILD_DIR) && \ export WINEPREFIX=$(BUILD_DIR) && \ cd $(BUILD_DIR) && \ - cp $(CURDIR)/src/python-$(PYTHON_VERSION).msi . && \ - wine msiexec /qn /i python-$(PYTHON_VERSION).msi && \ mkdir pip && \ cd pip && \ wine c:\\Python27\\python.exe -m pip download requests && \ @@ -81,11 +107,11 @@ new_packages: src/python-$(PYTHON_VERSION).msi wine c:\\Python27\\python.exe -m pip download pyOpenSSL && \ mkdir -p $(CURDIR)/src/pip && \ cp -r * $(CURDIR)/src/pip - rm -rf $(BUILD_DIR) src/vcredist_x86.exe: + mkdir -p src && \ cd src && \ - wget https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966… + curl -O https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966… setup: - sudo apt-get install scons upx-ucl wine + sudo apt-get install scons upx-ucl wine \ No newline at end of file diff --git a/agents/windows/make-chroot b/agents/windows/make-chroot index 288080a..d586ec1 100755 --- a/agents/windows/make-chroot +++ b/agents/windows/make-chroot @@ -10,7 +10,8 @@ MIRROR=http://de.archive.ubuntu.com/ubuntu export HOSTNAME=cmk-windows-agent-chroot export LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 -BASE="$(realpath ../..)" +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +BASE="$(realpath $SCRIPT_DIR/../..)" if [ -z $CHROOT_PATH ]; then echo "ERROR: CHROOT_PATH not set" @@ -55,7 +56,9 @@ package_names=( autoconf automake build-essential + cabextract ca-certificates + curl gcc gettext language-pack-en @@ -152,10 +155,10 @@ fi sed -ri 's/^_apt:x:[0-9]+:/_apt:x:0:/g' $CHROOT_PATH/etc/passwd # We need to enable 32bit Wine -./do-chroot "dpkg --add-architecture i386 && apt-get update && apt-get -y install wine32" +$SCRIPT_DIR/do-chroot "dpkg --add-architecture i386 && apt-get update && apt-get -y install wine32" # Build Google Test and Google Mock -./do-chroot "$CHROOT_BUILD_DIR/agents/windows/test/build-googletest" +$SCRIPT_DIR/do-chroot "$CHROOT_BUILD_DIR/agents/windows/test/build-googletest" cp "$BASE/agents/windows/$VERSIONFILE" "$CHROOT_PATH/"