Module: check_mk
Branch: master
Commit: c647025b6b00ac08f9be79daa694913ed6b4fcca
URL:
http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c647025b6b00ac…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Mon Mar 4 09:01:39 2019 +0100
7224 FIX Backup/Restore: Fix vanishing files terminating a backup
The "omd backup" mechanism recursively backups all site related
files of a Check_MK site. When files in a site directory, that
is currently being processed vanish, this could lead to failed
backups with errors like this:
C+:
Site backup failed: Traceback (most recent call last):
File "/omd/versions/1.5.0p8.cee/bin/omd", line 4553, in
command_function(args, command_options)
File "/omd/versions/1.5.0p8.cee/bin/omd", line 3711, in main_backup
backup_site_to_tarfile(fh, tar_mode, options)
File "/omd/versions/1.5.0p8.cee/bin/omd", line 3686, in backup_site_to_tarfile
backup_site_files_to_tarfile(tar, options)
File "/omd/versions/1.5.0p8.cee/bin/omd", line 3556, in
backup_site_files_to_tarfile
tar.add(g_sitedir, g_sitename, exclude=filter_files)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
recursive, exclude, filter)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
recursive, exclude, filter)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
recursive, exclude, filter)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
recursive, exclude, filter)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
recursive, exclude, filter)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2009, in add
tarinfo = self.gettarinfo(name, arcname)
File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 1881, in
gettarinfo
statres = os.lstat(name)
OSError: [Errno 2] No such file or directory:
'/omd/sites/xyz/var/pnp4nagios/perfdata/hastenichgesehn/Interface_BG4_2OG_VLAN_421.xml.new'
C-:
After this change the backup continues in such a situation
excluding the just vanished file.
Change-Id: Ic76bc983ebd221687e2f249bf6bcb3389f348aa2
---
.werks/7224 | 43 ++++++++++++++++++++
omd/packages/omd/omdlib/backup.py | 18 ++++++++-
tests/unit/omdlib/test_backup.py | 85 +++++++++++++++++++++++++++++++++++++++
3 files changed, 144 insertions(+), 2 deletions(-)
diff --git a/.werks/7224 b/.werks/7224
new file mode 100644
index 0000000..b067d0e
--- /dev/null
+++ b/.werks/7224
@@ -0,0 +1,43 @@
+Title: Backup/Restore: Fix vanishing files terminating a backup
+Level: 2
+Component: omd
+Compatible: compat
+Edition: cre
+Version: 1.6.0i1
+Date: 1551685160
+Class: fix
+
+The "omd backup" mechanism recursively backups all site related
+files of a Check_MK site. When files in a site directory, that
+is currently being processed vanish, this could lead to failed
+backups with errors like this:
+
+C+:
+Site backup failed: Traceback (most recent call last):
+File "/omd/versions/1.5.0p8.cee/bin/omd", line 4553, in
+command_function(args, command_options)
+File "/omd/versions/1.5.0p8.cee/bin/omd", line 3711, in main_backup
+backup_site_to_tarfile(fh, tar_mode, options)
+File "/omd/versions/1.5.0p8.cee/bin/omd", line 3686, in backup_site_to_tarfile
+backup_site_files_to_tarfile(tar, options)
+File "/omd/versions/1.5.0p8.cee/bin/omd", line 3556, in
backup_site_files_to_tarfile
+tar.add(g_sitedir, g_sitename, exclude=filter_files)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
+recursive, exclude, filter)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
+recursive, exclude, filter)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
+recursive, exclude, filter)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
+recursive, exclude, filter)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2032, in add
+recursive, exclude, filter)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 2009, in add
+tarinfo = self.gettarinfo(name, arcname)
+File "/omd/versions/1.5.0p8.cee/lib/python2.7/tarfile.py", line 1881, in
gettarinfo
+statres = os.lstat(name)
+OSError: [Errno 2] No such file or directory:
'/omd/sites/xyz/var/pnp4nagios/perfdata/hastenichgesehn/Interface_BG4_2OG_VLAN_421.xml.new'
+C-:
+
+After this change the backup continues in such a situation
+excluding the just vanished file.
diff --git a/omd/packages/omd/omdlib/backup.py b/omd/packages/omd/omdlib/backup.py
index 70ca46b..268574c 100644
--- a/omd/packages/omd/omdlib/backup.py
+++ b/omd/packages/omd/omdlib/backup.py
@@ -106,7 +106,7 @@ class BackupTarFile(tarfile.TarFile):
"""We need to use our tarfile class here to perform a rrdcached
SUSPEND/RESUME
to prevent writing to individual RRDs during backups."""
- def __init__(self, *args, **kwargs):
+ def __init__(self, name, mode, fileobj, **kwargs):
self._site = kwargs.pop("site")
self._verbose = kwargs.pop("verbose")
self._site_stopped = self._site.is_stopped()
@@ -114,7 +114,21 @@ class BackupTarFile(tarfile.TarFile):
self._sock = None
self._sites_path = os.path.realpath("/omd/sites")
- super(BackupTarFile, self).__init__(*args, **kwargs)
+ super(BackupTarFile, self).__init__(name, mode, fileobj, **kwargs)
+
+ # We override this function to workaround an issue in the builtin add() method in
+ # case it is called in recursive mode and a file vanishes between the os.listdir()
+ # and the first file access (often seen os.lstat()) during backup. Instead of
failing
+ # like this we want to skip those files silently during backup.
+ def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): #
pylint: disable=redefined-builtin
+ try:
+ super(BackupTarFile, self).add(name, arcname, recursive, exclude, filter)
+ except OSError as e:
+ if e.errno != errno.ENOENT or arcname == self._site.name:
+ raise
+
+ if self._verbose:
+ sys.stdout.write("Skipping vanished file: %s\n" % arcname)
def addfile(self, tarinfo, fileobj=None):
# In case of a stopped site or stopped rrdcached there is no
diff --git a/tests/unit/omdlib/test_backup.py b/tests/unit/omdlib/test_backup.py
new file mode 100644
index 0000000..d6b6942
--- /dev/null
+++ b/tests/unit/omdlib/test_backup.py
@@ -0,0 +1,85 @@
+# encoding: utf-8
+# pylint: disable=redefined-outer-name
+
+import tarfile
+import pytest # type: ignore
+from pathlib2 import Path
+import omdlib
+import omdlib.main
+import omdlib.backup
+
+
+(a)pytest.fixture()
+def site(tmp_path, monkeypatch):
+ monkeypatch.setattr(omdlib, "__version__", "1.3.3i7.cee")
+ omd_root = tmp_path / "site"
+ omd_root.mkdir(parents=True, exist_ok=True)
+ (omd_root / "version").symlink_to("../versions/%s" %
omdlib.__version__)
+
+ class UnitTestSite(omdlib.main.SiteContext):
+ @property
+ def dir(self):
+ return str(omd_root)
+
+ return UnitTestSite("unit")
+
+
+def test_backup_site_to_tarfile(site, tmp_path):
+ # Write some file for testing the backup procedure
+ with Path(site.dir + "/test123").open("w",
encoding="utf-8") as f:
+ f.write(u"uftauftauftata")
+
+ tar_path = tmp_path / "backup.tar"
+ with tar_path.open("wb") as backup_tar:
+ omdlib.backup.backup_site_to_tarfile(site, backup_tar, mode="w:",
options={}, verbose=False)
+
+ with tar_path.open("rb") as backup_tar:
+ tar = tarfile.open(fileobj=backup_tar, mode="r:*")
+ sitename, version = omdlib.backup.get_site_and_version_from_backup(tar)
+ assert sitename == "unit"
+ assert version == "1.3.3i7.cee"
+
+ names = [tarinfo.name for tarinfo in tar]
+ assert "unit/test123" in names
+
+
+def test_backup_site_to_tarfile_broken_link(site, tmp_path):
+ Path(site.dir + "/link").symlink_to("agag")
+
+ tar_path = tmp_path / "backup.tar"
+ with tar_path.open("wb") as backup_tar:
+ omdlib.backup.backup_site_to_tarfile(site, backup_tar, mode="w:",
options={}, verbose=False)
+
+ with tar_path.open("rb") as backup_tar:
+ tar = tarfile.open(fileobj=backup_tar, mode="r:*")
+ _sitename, _version = omdlib.backup.get_site_and_version_from_backup(tar)
+
+ link = tar.getmember("unit/link")
+ assert link.linkname == "agag"
+
+
+def test_backup_site_to_tarfile_vanishing_files(site, tmp_path, monkeypatch):
+ test_dir = Path(site.dir) / "xyz"
+ test_file = test_dir / "test_file"
+ test_dir.mkdir(parents=True, exist_ok=True) # pylint: disable=no-member
+ test_file.touch() # pylint: disable=no-member
+
+ orig_add = omdlib.backup.BackupTarFile.add
+
+ def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): #
pylint: disable=redefined-builtin
+ # The add() was called for test_dir which then calls os.listdir() and
+ # add() for all found entries. Remove the test_file here to simulate
+ # a vanished file during this step.
+ if arcname == "unit/xyz/test_file":
+ test_file.unlink() # pylint: disable=no-member
+ orig_add(self, name, arcname, recursive, exclude, filter)
+
+ monkeypatch.setattr(omdlib.backup.BackupTarFile, "add", add)
+
+ tar_path = tmp_path / "backup.tar"
+ with tar_path.open("wb") as backup_tar:
+ omdlib.backup.backup_site_to_tarfile(site, backup_tar, mode="w:",
options={}, verbose=False)
+
+ with tar_path.open("rb") as backup_tar:
+ tar = tarfile.open(fileobj=backup_tar, mode="r:*")
+ _sitename, _version = omdlib.backup.get_site_and_version_from_backup(tar)