[checkmk-commits] 7224 FIX Backup/Restore: Fix vanishing files terminating a backup

Module: check_mk Branch: master Commit: c647025b6b00ac08f9be79daa694913ed6b4fcca URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c647025b6b00ac… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Mon Mar 4 09:01:39 2019 +0100 7224 FIX Backup/Restore: Fix vanishing files terminating a backup The "omd backup" mechanism recursively backups all site related files of a Check_MK site. When files in a site directory, that is currently being processed vanish, this could lead to failed backups with errors like this: C+: Site backup failed: Traceback (most recent call last): File "/omd/versions/1.5.0p8.cee/bin/omd", line 4553, in command_function(args, command_options) File "/omd/versions/1.5.0p8.cee/bin/omd", line 3711, in main_backup backup_site_to_tarfile(fh, tar_mode, options) File "/omd/versions/1.5.0p8.cee/bin/omd", line 3686, in backup_site_to_tarfile backup_site_files_to_tarfile(tar, options) File "/omd/versions/1.5.0p8.cee/bin/omd", line 3556, in backup_site_files_to_tarfile tar.add(g_sitedir, g_sitename, exclude=filter_files) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add recursive, exclude, filter) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add recursive, exclude, filter) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add recursive, exclude, filter) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add recursive, exclude, filter) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add recursive, exclude, filter) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2009, in add tarinfo = self.gettarinfo(name, arcname) File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 1881, in gettarinfo statres = os.lstat(name) OSError: [Errno 2] No such file or directory: '/omd/sites/xyz/var/pnp4nagios/perfdata/hastenichgesehn/Interface_BG4_2OG_VLAN_421.xml.new' C-: After this change the backup continues in such a situation excluding the just vanished file. Change-Id: Ic76bc983ebd221687e2f249bf6bcb3389f348aa2 --- .werks/7224 | 43 ++++++++++++++++++++ omd/packages/omd/omdlib/backup.py | 18 ++++++++- tests/unit/omdlib/test_backup.py | 85 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/.werks/7224 b/.werks/7224 new file mode 100644 index 0000000..b067d0e --- /dev/null +++ b/.werks/7224 @@ -0,0 +1,43 @@ +Title: Backup/Restore: Fix vanishing files terminating a backup +Level: 2 +Component: omd +Compatible: compat +Edition: cre +Version: 1.6.0i1 +Date: 1551685160 +Class: fix + +The "omd backup" mechanism recursively backups all site related +files of a Check_MK site. When files in a site directory, that +is currently being processed vanish, this could lead to failed +backups with errors like this: + +C+: +Site backup failed: Traceback (most recent call last): +File "/omd/versions/1.5.0p8.cee/bin/omd", line 4553, in +command_function(args, command_options) +File "/omd/versions/1.5.0p8.cee/bin/omd", line 3711, in main_backup +backup_site_to_tarfile(fh, tar_mode, options) +File "/omd/versions/1.5.0p8.cee/bin/omd", line 3686, in backup_site_to_tarfile +backup_site_files_to_tarfile(tar, options) +File "/omd/versions/1.5.0p8.cee/bin/omd", line 3556, in backup_site_files_to_tarfile +tar.add(g_sitedir, g_sitename, exclude=filter_files) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add +recursive, exclude, filter) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add +recursive, exclude, filter) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add +recursive, exclude, filter) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add +recursive, exclude, filter) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add +recursive, exclude, filter) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2009, in add +tarinfo = self.gettarinfo(name, arcname) +File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 1881, in gettarinfo +statres = os.lstat(name) +OSError: [Errno 2] No such file or directory: '/omd/sites/xyz/var/pnp4nagios/perfdata/hastenichgesehn/Interface_BG4_2OG_VLAN_421.xml.new' +C-: + +After this change the backup continues in such a situation +excluding the just vanished file. diff --git a/omd/packages/omd/omdlib/backup.py b/omd/packages/omd/omdlib/backup.py index 70ca46b..268574c 100644 --- a/omd/packages/omd/omdlib/backup.py +++ b/omd/packages/omd/omdlib/backup.py @@ -106,7 +106,7 @@ class BackupTarFile(tarfile.TarFile): """We need to use our tarfile class here to perform a rrdcached SUSPEND/RESUME to prevent writing to individual RRDs during backups.""" - def __init__(self, *args, **kwargs): + def __init__(self, name, mode, fileobj, **kwargs): self._site = kwargs.pop("site") self._verbose = kwargs.pop("verbose") self._site_stopped = self._site.is_stopped() @@ -114,7 +114,21 @@ class BackupTarFile(tarfile.TarFile): self._sock = None self._sites_path = os.path.realpath("/omd/sites") - super(BackupTarFile, self).__init__(*args, **kwargs) + super(BackupTarFile, self).__init__(name, mode, fileobj, **kwargs) + + # We override this function to workaround an issue in the builtin add() method in + # case it is called in recursive mode and a file vanishes between the os.listdir() + # and the first file access (often seen os.lstat()) during backup. Instead of failing + # like this we want to skip those files silently during backup. + def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): # pylint: disable=redefined-builtin + try: + super(BackupTarFile, self).add(name, arcname, recursive, exclude, filter) + except OSError as e: + if e.errno != errno.ENOENT or arcname == self._site.name: + raise + + if self._verbose: + sys.stdout.write("Skipping vanished file: %s\n" % arcname) def addfile(self, tarinfo, fileobj=None): # In case of a stopped site or stopped rrdcached there is no diff --git a/tests/unit/omdlib/test_backup.py b/tests/unit/omdlib/test_backup.py new file mode 100644 index 0000000..d6b6942 --- /dev/null +++ b/tests/unit/omdlib/test_backup.py @@ -0,0 +1,85 @@ +# encoding: utf-8 +# pylint: disable=redefined-outer-name + +import tarfile +import pytest # type: ignore +from pathlib2 import Path +import omdlib +import omdlib.main +import omdlib.backup + + +(a)pytest.fixture() +def site(tmp_path, monkeypatch): + monkeypatch.setattr(omdlib, "__version__", "1.3.3i7.cee") + omd_root = tmp_path / "site" + omd_root.mkdir(parents=True, exist_ok=True) + (omd_root / "version").symlink_to("../versions/%s" % omdlib.__version__) + + class UnitTestSite(omdlib.main.SiteContext): + @property + def dir(self): + return str(omd_root) + + return UnitTestSite("unit") + + +def test_backup_site_to_tarfile(site, tmp_path): + # Write some file for testing the backup procedure + with Path(site.dir + "/test123").open("w", encoding="utf-8") as f: + f.write(u"uftauftauftata") + + tar_path = tmp_path / "backup.tar" + with tar_path.open("wb") as backup_tar: + omdlib.backup.backup_site_to_tarfile(site, backup_tar, mode="w:", options={}, verbose=False) + + with tar_path.open("rb") as backup_tar: + tar = tarfile.open(fileobj=backup_tar, mode="r:*") + sitename, version = omdlib.backup.get_site_and_version_from_backup(tar) + assert sitename == "unit" + assert version == "1.3.3i7.cee" + + names = [tarinfo.name for tarinfo in tar] + assert "unit/test123" in names + + +def test_backup_site_to_tarfile_broken_link(site, tmp_path): + Path(site.dir + "/link").symlink_to("agag") + + tar_path = tmp_path / "backup.tar" + with tar_path.open("wb") as backup_tar: + omdlib.backup.backup_site_to_tarfile(site, backup_tar, mode="w:", options={}, verbose=False) + + with tar_path.open("rb") as backup_tar: + tar = tarfile.open(fileobj=backup_tar, mode="r:*") + _sitename, _version = omdlib.backup.get_site_and_version_from_backup(tar) + + link = tar.getmember("unit/link") + assert link.linkname == "agag" + + +def test_backup_site_to_tarfile_vanishing_files(site, tmp_path, monkeypatch): + test_dir = Path(site.dir) / "xyz" + test_file = test_dir / "test_file" + test_dir.mkdir(parents=True, exist_ok=True) # pylint: disable=no-member + test_file.touch() # pylint: disable=no-member + + orig_add = omdlib.backup.BackupTarFile.add + + def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): # pylint: disable=redefined-builtin + # The add() was called for test_dir which then calls os.listdir() and + # add() for all found entries. Remove the test_file here to simulate + # a vanished file during this step. + if arcname == "unit/xyz/test_file": + test_file.unlink() # pylint: disable=no-member + orig_add(self, name, arcname, recursive, exclude, filter) + + monkeypatch.setattr(omdlib.backup.BackupTarFile, "add", add) + + tar_path = tmp_path / "backup.tar" + with tar_path.open("wb") as backup_tar: + omdlib.backup.backup_site_to_tarfile(site, backup_tar, mode="w:", options={}, verbose=False) + + with tar_path.open("rb") as backup_tar: + tar = tarfile.open(fileobj=backup_tar, mode="r:*") + _sitename, _version = omdlib.backup.get_site_and_version_from_backup(tar)