From 381b8a2fbeadaf9799f09b1d5f213c7586b27588 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Fri, 8 Sep 2017 16:44:57 +0200 Subject: [PATCH 01/13] Updated TODO New ideas for kvmBackup --- TODO | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/TODO b/TODO index 7873a2b..e59622e 100644 --- a/TODO +++ b/TODO @@ -4,3 +4,7 @@ * verbosity level * copy CD-rom data * testing wrong configurations +* rotating using dates, not numbers +* dealing with power off VM (delete snapshot will not update domain) +* deal with empty configurations +* python packaging? From 6ba458fb48f1052d046d9e82e9dc17686fa7b726 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Thu, 3 Feb 2022 17:40:44 +0100 Subject: [PATCH 02/13] :see_no_evil: update gitignore --- .gitignore | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index bc38d11..a8ff487 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ __pycache__/ # Distribution / packaging .Python -env/ build/ develop-eggs/ dist/ @@ -20,9 +19,12 @@ lib64/ parts/ sdist/ var/ +wheels/ +share/python-wheels/ *.egg-info/ .installed.cfg *.egg +MANIFEST # PyInstaller # Usually these files are written by a python script from a template @@ -37,12 +39,17 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ +.nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml -*,cover +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ # Translations *.mo @@ -50,13 +57,99 @@ coverage.xml # Django stuff: *.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy # Sphinx documentation docs/_build/ # PyBuilder +.pybuilder/ target/ +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + # backup files *~ - From 1a515eb96c14284512fe5a6fc3db0e0ff009d91a Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Thu, 3 Feb 2022 17:42:41 +0100 Subject: [PATCH 03/13] :rotating_light: run autopep8 and isort --- Lib/flock.py | 39 ++++---- Lib/helper.py | 240 ++++++++++++++++++++++++++++---------------------- kvmBackup.py | 191 ++++++++++++++++++++------------------- 3 files changed, 258 insertions(+), 212 deletions(-) diff --git a/Lib/flock.py b/Lib/flock.py index 3db2547..b30794f 100644 --- a/Lib/flock.py +++ b/Lib/flock.py @@ -25,31 +25,36 @@ """ +import logging import os import socket -import logging # Logging istance logger = logging.getLogger(__name__) + class flock(object): '''Class to handle creating and removing (pid) lockfiles''' # custom exceptions - class FileLockAcquisitionError(Exception): pass - class FileLockReleaseError(Exception): pass + class FileLockAcquisitionError(Exception): + pass + + class FileLockReleaseError(Exception): + pass # convenience callables for formatting - addr = lambda self: '%d@%s' % (self.pid, self.host) - fddr = lambda self: '<%s %s>' % (self.path, self.addr()) - pddr = lambda self, lock: '<%s %s@%s>' %\ - (self.path, lock['pid'], lock['host']) + def addr(self): return '%d@%s' % (self.pid, self.host) + def fddr(self): return '<%s %s>' % (self.path, self.addr()) + + def pddr(self, lock): return '<%s %s@%s>' %\ + (self.path, lock['pid'], lock['host']) def __init__(self, path, debug=None): - self.pid = os.getpid() - self.host = socket.gethostname() - self.path = path - self.debug = debug # set this to get status messages + self.pid = os.getpid() + self.host = socket.gethostname() + self.path = path + self.debug = debug # set this to get status messages def acquire(self): '''Acquire a lock, returning self if successful, False otherwise''' @@ -90,7 +95,7 @@ def _readlock(self): '''Internal method to read lock info''' try: lock = {} - fh = open(self.path) + fh = open(self.path) data = fh.read().rstrip().split('@') fh.close() lock['pid'], lock['host'] = data @@ -116,15 +121,17 @@ def __del__(self): '''Magic method to clean up lock when program exits''' self.release() -## ======== +# ======== + +# Test programs: run test1.py then test2.py (in the same dir) +# from another teminal -- test2.py should print +# a message that there is a lock in place and exit. -## Test programs: run test1.py then test2.py (in the same dir) -## from another teminal -- test2.py should print -## a message that there is a lock in place and exit. if __name__ == "__main__": # test1.py from time import sleep + #from flock import flock lock = flock('tmp.lock', True).acquire() if lock: diff --git a/Lib/helper.py b/Lib/helper.py index c9d4c06..40ce5c8 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -25,18 +25,18 @@ """ +import logging import os -import uuid import shlex import shutil import signal -import libvirt -import logging import subprocess - +import uuid # To inspect xml import xml.etree.ElementTree as ET +import libvirt + # Logging istance logger = logging.getLogger(__name__) @@ -45,97 +45,104 @@ # una funzione che ho trovato qui: https://blog.nelhage.com/2010/02/a-very-subtle-bug/ # e che dovrebbe gestire i segnali strani quando esco da un suprocess -preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def preexec_fn(): return signal.signal(signal.SIGPIPE, signal.SIG_DFL) + def dumpXML(domain, path): """DumpXML inside PATH""" - logger.info("Dumping XMLs for domain %s" %(domain.name())) + logger.info("Dumping XMLs for domain %s" % (domain.name())) - #I need to return wrote files + # I need to return wrote files xml_files = [] - dest_file = "%s.xml" %(domain.name()) + dest_file = "%s.xml" % (domain.name()) dest_file = os.path.join(path, dest_file) if os.path.exists(dest_file): - raise Exception, "File %s exists!!" %(dest_file) + raise Exception, "File %s exists!!" % (dest_file) dest_fh = open(dest_file, "w") - #dump different xmls files. First of all, the offline dump + # dump different xmls files. First of all, the offline dump xml = domain.XMLDesc() dest_fh.write(xml) dest_fh.close() xml_files += [dest_file] - logger.debug("File %s wrote" %(dest_file)) + logger.debug("File %s wrote" % (dest_file)) - #All flags: libvirt.VIR_DOMAIN_XML_INACTIVE, libvirt.VIR_DOMAIN_XML_MIGRATABLE, libvirt.VIR_DOMAIN_XML_SECURE, libvirt.VIR_DOMAIN_XML_UPDATE_CPU - dest_file = "%s-inactive.xml" %(domain.name()) + # All flags: libvirt.VIR_DOMAIN_XML_INACTIVE, libvirt.VIR_DOMAIN_XML_MIGRATABLE, libvirt.VIR_DOMAIN_XML_SECURE, libvirt.VIR_DOMAIN_XML_UPDATE_CPU + dest_file = "%s-inactive.xml" % (domain.name()) dest_file = os.path.join(path, dest_file) if os.path.exists(dest_file): - raise Exception, "File %s exists!!" %(dest_file) + raise Exception, "File %s exists!!" % (dest_file) dest_fh = open(dest_file, "w") - #dump different xmls files. First of all, the offline dump + # dump different xmls files. First of all, the offline dump xml = domain.XMLDesc(flags=libvirt.VIR_DOMAIN_XML_INACTIVE) dest_fh.write(xml) dest_fh.close() xml_files += [dest_file] - logger.debug("File %s wrote" %(dest_file)) + logger.debug("File %s wrote" % (dest_file)) - #Dump a migrate config file - dest_file = "%s-migratable.xml" %(domain.name()) + # Dump a migrate config file + dest_file = "%s-migratable.xml" % (domain.name()) dest_file = os.path.join(path, dest_file) if os.path.exists(dest_file): - raise Exception, "File %s exists!!" %(dest_file) + raise Exception, "File %s exists!!" % (dest_file) dest_fh = open(dest_file, "w") - #dump different xmls files. First of all, the offline dump - xml = domain.XMLDesc(flags=libvirt.VIR_DOMAIN_XML_INACTIVE+libvirt.VIR_DOMAIN_XML_MIGRATABLE) + # dump different xmls files. First of all, the offline dump + xml = domain.XMLDesc( + flags=libvirt.VIR_DOMAIN_XML_INACTIVE+libvirt.VIR_DOMAIN_XML_MIGRATABLE) dest_fh.write(xml) dest_fh.close() xml_files += [dest_file] - logger.debug("File %s wrote" %(dest_file)) + logger.debug("File %s wrote" % (dest_file)) return xml_files -#Define a function to get all disk for a certain domain +# Define a function to get all disk for a certain domain + + def getDisks(domain): """Get al disks from a particoular domain""" - #the fromstring method returns the root node + # the fromstring method returns the root node root = ET.fromstring(domain.XMLDesc()) - #then use XPath to search a line like under tag + # then use XPath to search a line like under tag devices = root.findall("./devices/disk[@device='disk']") - #Now find the child element with source tag + # Now find the child element with source tag sources = [device.find("source").attrib for device in devices] - #get also dev target + # get also dev target targets = [device.find("target").attrib for device in devices] - #iterate amoung sources and targets + # iterate amoung sources and targets if len(sources) != len(targets): - raise Exception, "Targets and sources lengths are different %s:%s" %(len(sources), len(targets)) + raise Exception, "Targets and sources lengths are different %s:%s" % (len(sources), len(targets)) - #here all the devices I want to back up + # here all the devices I want to back up devs = {} for i in range(len(sources)): devs[targets[i]["dev"]] = sources[i]["file"] - #return dev, file path list + # return dev, file path list return devs + class Snapshot(): """A class to deal with libvirt snapshot""" @@ -160,30 +167,30 @@ def getDomain(self): def getDisks(self): """Call getDisk on my instance""" - #get my domain + # get my domain domain = self.getDomain() - #call getDisk to get the disks to do snapshot + # call getDisk to get the disks to do snapshot return getDisks(domain) def dumpXML(self, path): """Call dumpXML on my instance""" - #get my domain + # get my domain domain = self.getDomain() - #call getDisk to get the disks to do snapshot + # call getDisk to get the disks to do snapshot return dumpXML(domain, path) - + def hasCurrentSnapshot(self): """call hasCurrentSnapshot on domain class attribute""" - - #get my domain + + # get my domain domain = self.getDomain() - + if domain.hasCurrentSnapshot() == 0: return False - + else: return True @@ -191,40 +198,45 @@ def getSnapshotXML(self): """Since I need to do a Snapshot with a XML file, I will create an XML to call the appropriate libvirt method""" - #get my domain + # get my domain domain = self.getDomain() - #call getDisk to get the disks to do snapshot + # call getDisk to get the disks to do snapshot self.disks = self.getDisks() - #get a snapshot id + # get a snapshot id self.snapshotId = str(uuid.uuid1()).split("-")[0] - #now construct all diskspec + # now construct all diskspec diskspecs = [] for disk in self.disks.iterkeys(): - diskspecs += ["--diskspec %s,file=/var/lib/libvirt/images/snapshot_%s_%s-%s.img" %(disk, self.domain_name, disk, self.snapshotId)] + diskspecs += ["--diskspec %s,file=/var/lib/libvirt/images/snapshot_%s_%s-%s.img" % + (disk, self.domain_name, disk, self.snapshotId)] - my_cmd = "virsh snapshot-create-as --domain {domain_name} {snapshotId} {diskspecs} --disk-only --atomic --quiesce --print-xml".format(domain_name=domain.name(), snapshotId=self.snapshotId, diskspecs=" ".join(diskspecs)) + my_cmd = "virsh snapshot-create-as --domain {domain_name} {snapshotId} {diskspecs} --disk-only --atomic --quiesce --print-xml".format( + domain_name=domain.name(), snapshotId=self.snapshotId, diskspecs=" ".join(diskspecs)) - logger.debug("Executing: %s" %(my_cmd)) + logger.debug("Executing: %s" % (my_cmd)) - #split the executable + # split the executable my_cmds = shlex.split(my_cmd) - #Launch command - create_xml = subprocess.Popen(my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) + # Launch command + create_xml = subprocess.Popen( + my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) - #read output in xml + # read output in xml self.snapshot_xml = create_xml.stdout.read() - #Lancio il comando e aspetto che termini + # Lancio il comando e aspetto che termini status = create_xml.wait() if status != 0: - logger.error("Error for %s:%s" %(my_cmds, create_xml.stderr.read())) - logger.critical("{exe} returned {stato} state".format(stato=status, exe=my_cmds[0])) + logger.error("Error for %s:%s" % + (my_cmds, create_xml.stderr.read())) + logger.critical("{exe} returned {stato} state".format( + stato=status, exe=my_cmds[0])) raise Exception, "snapshot-create-as didn't work properly" return self.snapshot_xml @@ -232,72 +244,80 @@ def getSnapshotXML(self): def callSnapshot(self): """Create a snapshot for domain""" - #Don't redo a snapshot on the same item + # Don't redo a snapshot on the same item if self.snapshot is not None: logger.error("A snapshot is already defined for this domain") logger.warn("Returning the current snapshot") return self.snapshot - #i need a xml file for the domain + # i need a xml file for the domain if self.snapshot_xml is None: self.getSnapshotXML() - #Those are the flags I need for creating SnapShot - [disk_only, atomic, quiesce] = [libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_ATOMIC, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE] + # Those are the flags I need for creating SnapShot + [disk_only, atomic, quiesce] = [libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY, + libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_ATOMIC, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE] - #get a domain + # get a domain domain = self.getDomain() - #do a snapshot - logger.info("Creating snapshot %s for %s" %(self.snapshotId, self.domain_name)) - self.snapshot = domain.snapshotCreateXML(self.snapshot_xml, flags=sum([disk_only, atomic, quiesce])) + # do a snapshot + logger.info("Creating snapshot %s for %s" % + (self.snapshotId, self.domain_name)) + self.snapshot = domain.snapshotCreateXML( + self.snapshot_xml, flags=sum([disk_only, atomic, quiesce])) - #Once i've created a snapshot, I can read disks to have snapshot image name + # Once i've created a snapshot, I can read disks to have snapshot image name self.snapshot_disk = self.getDisks() - #debug + # debug for disk, top in self.snapshot_disk.iteritems(): - logger.debug("Created top image {top} for {domain_name} {disk}".format(top=top, domain_name=domain.name(), disk=disk)) + logger.debug("Created top image {top} for {domain_name} {disk}".format( + top=top, domain_name=domain.name(), disk=disk)) return self.snapshot def doBlockCommit(self): """Do a blockcommit for every disks shapshotted""" - #get a domain + # get a domain domain = self.getDomain() - logger.info("Blockcommitting %s" %(domain.name())) + logger.info("Blockcommitting %s" % (domain.name())) - #A blockcommit for every disks. Using names like libvirt variables. Base is the original image file + # A blockcommit for every disks. Using names like libvirt variables. Base is the original image file for disk in self.disks.iterkeys(): - #the command to execute - my_cmd = "virsh blockcommit {domain_name} {disk} --active --verbose --pivot".format(domain_name=domain.name(), disk=disk) - logger.debug("Executing: %s" %(my_cmd)) + # the command to execute + my_cmd = "virsh blockcommit {domain_name} {disk} --active --verbose --pivot".format( + domain_name=domain.name(), disk=disk) + logger.debug("Executing: %s" % (my_cmd)) - #split the executable + # split the executable my_cmds = shlex.split(my_cmd) - #Launch command - blockcommit = subprocess.Popen(my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) + # Launch command + blockcommit = subprocess.Popen( + my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) - #read output throug processing + # read output throug processing for line in blockcommit.stdout: line = line.strip() if len(line) is 0: continue - logger.debug("%s" %(line)) + logger.debug("%s" % (line)) - #Lancio il comando e aspetto che termini + # Lancio il comando e aspetto che termini status = blockcommit.wait() if status != 0: - logger.error("Error for %s:%s" %(my_cmds, blockcommit.stderr.read())) - logger.critical("{exe} returned {stato} state".format(stato=status, exe=my_cmds[0])) + logger.error("Error for %s:%s" % + (my_cmds, blockcommit.stderr.read())) + logger.critical("{exe} returned {stato} state".format( + stato=status, exe=my_cmds[0])) raise Exception, "blockcommit didn't work properly" - #After blockcommit, I need to check that image were successfully pivoted + # After blockcommit, I need to check that image were successfully pivoted test_disks = self.getDisks() for disk, base in self.disks.iteritems(): @@ -305,15 +325,16 @@ def doBlockCommit(self): top = self.snapshot_disk[disk] if base == test_base and top != test_base: - #I can remove the snapshotted image - logger.debug("Removing %s" %(top)) + # I can remove the snapshotted image + logger.debug("Removing %s" % (top)) os.remove(top) else: - logger.error("original base: %s, top: %s, new_base: %s" %(base, top, test_base)) - raise Exception, "Something goes wrong for snaphost %s" %(self.snapshotId) + logger.error("original base: %s, top: %s, new_base: %s" % + (base, top, test_base)) + raise Exception, "Something goes wrong for snaphost %s" % (self.snapshotId) - #If I arrive here, I can delete snapshot + # If I arrive here, I can delete snapshot self.__snapshotDelete() def __snapshotDelete(self): @@ -321,50 +342,55 @@ def __snapshotDelete(self): [metadata] = [libvirt.VIR_DOMAIN_SNAPSHOT_DELETE_METADATA_ONLY] - logger.info("Removing snapshot %s" %(self.snapshotId)) + logger.info("Removing snapshot %s" % (self.snapshotId)) self.snapshot.delete(flags=sum([metadata])) -#from https://bitbucket.org/russellballestrini/virt-back -def rotate( target, retention = 3 ): +# from https://bitbucket.org/russellballestrini/virt-back + + +def rotate(target, retention=3): """file rotation routine""" - - for i in range( retention-2, 0, -1 ): # count backwards - old_name = "%s.%s" % ( target, i ) - new_name = "%s.%s" % ( target, i + 1 ) + + for i in range(retention-2, 0, -1): # count backwards + old_name = "%s.%s" % (target, i) + new_name = "%s.%s" % (target, i + 1) if os.path.exists(old_name): - logger.debug("Moving %s into %s" %(old_name, new_name)) - shutil.move( old_name, new_name) + logger.debug("Moving %s into %s" % (old_name, new_name)) + shutil.move(old_name, new_name) - #Moving the first file + # Moving the first file if os.path.exists(target): - logger.debug("Moving %s into %s.1" %(target, target)) - shutil.move( target, target + '.1' ) + logger.debug("Moving %s into %s.1" % (target, target)) + shutil.move(target, target + '.1') + def packArchive(target): """Launch pigz for compressing files""" - my_cmd = "pigz --best --processes 8 %s" %(target) - logger.debug("Executing: %s" %(my_cmd)) + my_cmd = "pigz --best --processes 8 %s" % (target) + logger.debug("Executing: %s" % (my_cmd)) - #split the executable + # split the executable my_cmds = shlex.split(my_cmd) - #Launch command - pigz = subprocess.Popen(my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) + # Launch command + pigz = subprocess.Popen(my_cmds, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) - #read output throug processing + # read output throug processing for line in pigz.stdout: line = line.strip() if len(line) is 0: continue - logger.debug("%s" %(line)) + logger.debug("%s" % (line)) - #Lancio il comando e aspetto che termini + # Lancio il comando e aspetto che termini status = pigz.wait() if status != 0: - logger.error("Error for %s:%s" %(my_cmds, pigz.stderr.read())) - logger.critical("{exe} returned {stato} state".format(stato=status, exe=my_cmds[0])) + logger.error("Error for %s:%s" % (my_cmds, pigz.stderr.read())) + logger.critical("{exe} returned {stato} state".format( + stato=status, exe=my_cmds[0])) raise Exception, "pigz didn't work properly" diff --git a/kvmBackup.py b/kvmBackup.py index dd8df67..53288e6 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -24,25 +24,27 @@ """ +import argparse +import datetime +import logging import os -import sys -import yaml import shutil import socket -import libvirt -import logging +import sys import tarfile -import argparse -import datetime -#my functions -from Lib import helper, flock +import libvirt +import yaml + +# my functions +from Lib import flock, helper # the program name prog_name = os.path.basename(sys.argv[0]) # Logging istance -logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) logger = logging.getLogger(prog_name) notice = """ @@ -54,22 +56,26 @@ """ # A function to open a config file + + def loadConf(file_conf): config = yaml.load(open(file_conf)) - #read my defined domains + # read my defined domains hostname = socket.gethostname() hostname = hostname.split(".")[0] - #try to parse useful data + # try to parse useful data mydomains = config[hostname]["domains"] - #get my backup directory + # get my backup directory backupdir = config[hostname]["backupdir"] return mydomains, backupdir, config -#a function to check current day of the week +# a function to check current day of the week + + def checkDay(day): now = datetime.datetime.now() today = now.strftime("%a") @@ -79,126 +85,128 @@ def checkDay(day): return False + def filterDomains(domains, user_domains): """filter domamin by user domains""" # Those are user domains (as a list) user_domains = [domain.strip() for domain in user_domains.split(",")] found_domains = [] - + # test for domain existances for domain in user_domains: if domain not in domains: - logger.error("User domain '%s' not found" %(domain)) - + logger.error("User domain '%s' not found" % (domain)) + else: found_domains += [domain] - + # Now return the filtered domains return found_domains - + def backup(domain, parameters, backupdir): """Do all the operation needed for backup""" - + # create a snapshot instance snapshot = helper.Snapshot(domain) - + # check if no snapshot are defined if snapshot.hasCurrentSnapshot() is True: - raise Exception, "Domain '%s' has already a snapshot" %(domain) - - #changing directory + raise Exception, "Domain '%s' has already a snapshot" % (domain) + + # changing directory olddir = os.getcwd() workdir = os.path.join(backupdir, domain) - #creating directory if not exists + # creating directory if not exists if not os.path.exists(workdir) and not os.path.isdir(workdir): - logger.info("Creating directory '%s'" %(workdir)) + logger.info("Creating directory '%s'" % (workdir)) os.mkdir(workdir) - #cange directory + # cange directory os.chdir(workdir) - #a timestamp directory in which to put files + # a timestamp directory in which to put files date = datetime.datetime.now().strftime('%Y-%m-%d') datadir = os.path.join(workdir, date) - #creating datadir - logger.debug("Creating directory '%s'" %(datadir)) + # creating datadir + logger.debug("Creating directory '%s'" % (datadir)) os.mkdir(datadir) - #define the target backup + # define the target backup ext, tar_mode = '.tar', 'w' tar_name = domain + ext - tar_path = os.path.join( workdir, tar_name ) + tar_path = os.path.join(workdir, tar_name) tar_path_gz = tar_path + ".gz" - #call rotation directive - if os.path.isfile( tar_path_gz ): # if file exists, run rotate - logger.info('rotating backup files for ' + domain ) - helper.rotate( tar_path_gz, parameters["rotate"] ) + # call rotation directive + if os.path.isfile(tar_path_gz): # if file exists, run rotate + logger.info('rotating backup files for ' + domain) + helper.rotate(tar_path_gz, parameters["rotate"]) - tar = tarfile.open( tar_path, tar_mode ) + tar = tarfile.open(tar_path, tar_mode) - #call dumpXML + # call dumpXML xml_files = snapshot.dumpXML(path=datadir) - #Add xmlsto archive, and remove original file - logger.info("Adding XMLs files for domain '%s' to archive '%s'" %(domain, tar_path)) + # Add xmlsto archive, and remove original file + logger.info("Adding XMLs files for domain '%s' to archive '%s'" % + (domain, tar_path)) for xml_file in xml_files: - #backup file with its relative path + # backup file with its relative path xml_file = os.path.basename(xml_file) xml_file = os.path.join(date, xml_file) tar.add(xml_file) - logger.debug("'%s' added" %(xml_file)) + logger.debug("'%s' added" % (xml_file)) - logger.debug("removing '%s' from '%s'" %(xml_file, datadir)) + logger.debug("removing '%s' from '%s'" % (xml_file, datadir)) os.remove(xml_file) - - #call snapshot + # call snapshot snapshot.callSnapshot() - logger.info("Adding image files for '%s' to archive '%s'" %(domain, tar_path)) + logger.info("Adding image files for '%s' to archive '%s'" % + (domain, tar_path)) - #copying file + # copying file for disk, source in snapshot.disks.iteritems(): dest = os.path.join(datadir, os.path.basename(source)) - logger.debug("copying '%s' to '%s'" %(source, dest)) - shutil.copy2( source, dest ) + logger.debug("copying '%s' to '%s'" % (source, dest)) + shutil.copy2(source, dest) - #backup file with its relative path + # backup file with its relative path img_file = os.path.basename(dest) img_file = os.path.join(date, img_file) - logger.debug("Adding '%s' to archive '%s'" %(img_file, tar_path)) + logger.debug("Adding '%s' to archive '%s'" % (img_file, tar_path)) tar.add(img_file) - logger.debug("removing '%s' from '%s'" %(img_file, datadir)) + logger.debug("removing '%s' from '%s'" % (img_file, datadir)) os.remove(img_file) - #block commit (and delete snapshot) + # block commit (and delete snapshot) snapshot.doBlockCommit() - #closing archive + # closing archive tar.close() - #Now launcing subprocess with pigz - logger.info("Compressing '%s'" %(tar_name)) + # Now launcing subprocess with pigz + logger.info("Compressing '%s'" % (tar_name)) helper.packArchive(target=tar_name) - #revoving EMPTY datadir - logger.debug("removing '%s'" %(datadir)) + # revoving EMPTY datadir + logger.debug("removing '%s'" % (datadir)) os.rmdir(datadir) - #return to the original directory + # return to the original directory os.chdir(olddir) - logger.info("Backup for '%s' completed" %(domain)) + logger.info("Backup for '%s' completed" % (domain)) # A global connection instance @@ -206,20 +214,23 @@ def backup(domain, parameters, backupdir): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Backup of KVM-qcow2 domains') - parser.add_argument("-c", "--config", required=True, type=str, help="The config file") - parser.add_argument("--force", required=False, action='store_true', default=False, help="Force backup (with rotation)") - parser.add_argument("--domains", required=False, type=str, help="comma separated list of domains to backup ('virsh list --all' to get domains)") + parser.add_argument("-c", "--config", required=True, + type=str, help="The config file") + parser.add_argument("--force", required=False, action='store_true', + default=False, help="Force backup (with rotation)") + parser.add_argument("--domains", required=False, type=str, + help="comma separated list of domains to backup ('virsh list --all' to get domains)") args = parser.parse_args() - #logging notice + # logging notice sys.stderr.write(notice) sys.stderr.flush() - + # a flat to test if there were errors flag_errors = False - #Starting software - logger.info("Starting '%s'" %(prog_name)) + # Starting software + logger.info("Starting '%s'" % (prog_name)) lockfile = os.path.splitext(os.path.basename(sys.argv[0]))[0] + ".lock" lockfile_path = os.path.join("/var/run", lockfile) @@ -227,59 +238,61 @@ def backup(domain, parameters, backupdir): lock = flock.flock(lockfile_path, True).acquire() if not lock: - logger.error("Another istance of '%s' is running. Please wait for its termination or kill the running application" %(sys.argv[0])) + logger.error( + "Another istance of '%s' is running. Please wait for its termination or kill the running application" % (sys.argv[0])) sys.exit(-1) - #get all domain names + # get all domain names domains = [domain.name() for domain in conn.listAllDomains()] - + # filter domains with user provides domains (if needed) if args.domains is not None: - logger.info("Checking '%s' domains" %(args.domains)) - domains = filterDomains(domains,args.domains) + logger.info("Checking '%s' domains" % (args.domains)) + domains = filterDomains(domains, args.domains) - #parse configuration file + # parse configuration file mydomains, backupdir, config = loadConf(args.config) - #test for directory existance + # test for directory existance if not os.path.exists(backupdir) and os.path.isdir(backupdir) is False: - logger.info("Creating directory '%s'" %(backupdir)) + logger.info("Creating directory '%s'" % (backupdir)) os.mkdir(backupdir) - #debug - #pprint.pprint(mydomains) + # debug + # pprint.pprint(mydomains) for domain_name, parameters in mydomains.iteritems(): - #check if bakcup is needed + # check if bakcup is needed domain_backup = False - - #check if configuration domain exists or was filtered out + + # check if configuration domain exists or was filtered out if domain_name not in domains: - logger.warn("Ignoring domain '%s'" %(domain_name)) + logger.warn("Ignoring domain '%s'" % (domain_name)) continue for day in parameters["day_of_week"]: if checkDay(day) is True or args.force is True: - logger.info("Ready for backup of '%s'" %(domain_name)) + logger.info("Ready for backup of '%s'" % (domain_name)) domain_backup = True - #do backup stuff + # do backup stuff try: backup(domain_name, parameters, backupdir) - + except Exception, message: logger.exception(message) - logger.error("Domain '%s' was not backed up" %(domain_name)) + logger.error("Domain '%s' was not backed up" % + (domain_name)) flag_errors = True - #breaking cicle + # breaking cicle break if domain_backup is False: - logger.debug("Ignoring '%s' domain" %(domain_name)) + logger.debug("Ignoring '%s' domain" % (domain_name)) - #end of the program + # end of the program if flag_errors is False: - logger.info("'%s' completed successfully" %(prog_name)) + logger.info("'%s' completed successfully" % (prog_name)) else: - logger.warn("'%s' completed with error(s)" %(prog_name)) + logger.warn("'%s' completed with error(s)" % (prog_name)) From 5bfc4b9c94b61a25e96d897884a15a7540c0cde6 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Mon, 7 Feb 2022 13:03:37 +0100 Subject: [PATCH 04/13] :sparkles: support python3 importing from future and support code in python3 --- Lib/__init__.py | 2 +- Lib/flock.py | 13 ++++++++----- Lib/helper.py | 34 +++++++++++++++++++++++----------- kvmBackup.py | 6 ++++-- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/Lib/__init__.py b/Lib/__init__.py index a90b2d6..4e41f4a 100644 --- a/Lib/__init__.py +++ b/Lib/__init__.py @@ -26,5 +26,5 @@ import flock __author__ = "Paolo Cozzi" -__version__ = "1.0" +__version__ = "1.1" __all__ = ["helper", "flock"] diff --git a/Lib/flock.py b/Lib/flock.py index b30794f..f4a07b2 100644 --- a/Lib/flock.py +++ b/Lib/flock.py @@ -21,10 +21,13 @@ @author: Paolo Cozzi -Simple lockfile to detect previous instances of app (Python recipe http://code.activestate.com/recipes/498171/) +Simple lockfile to detect previous instances of app (Python recipe +http://code.activestate.com/recipes/498171/) """ +from __future__ import print_function + import logging import os import socket @@ -132,17 +135,17 @@ def __del__(self): # test1.py from time import sleep - #from flock import flock + # from flock import flock lock = flock('tmp.lock', True).acquire() if lock: sleep(30) else: - print 'locked!' + print('locked!') # test2.py from flock import flock lock = flock('tmp.lock', True).acquire() if lock: - print 'doing stuff' + print('doing stuff') else: - print 'locked!' + print('locked!') diff --git a/Lib/helper.py b/Lib/helper.py index 40ce5c8..008c421 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -25,6 +25,8 @@ """ +from __future__ import print_function + import logging import os import shlex @@ -32,6 +34,7 @@ import signal import subprocess import uuid + # To inspect xml import xml.etree.ElementTree as ET @@ -62,7 +65,7 @@ def dumpXML(domain, path): dest_file = os.path.join(path, dest_file) if os.path.exists(dest_file): - raise Exception, "File %s exists!!" % (dest_file) + raise Exception("File %s exists!!" % (dest_file)) dest_fh = open(dest_file, "w") @@ -74,12 +77,14 @@ def dumpXML(domain, path): xml_files += [dest_file] logger.debug("File %s wrote" % (dest_file)) - # All flags: libvirt.VIR_DOMAIN_XML_INACTIVE, libvirt.VIR_DOMAIN_XML_MIGRATABLE, libvirt.VIR_DOMAIN_XML_SECURE, libvirt.VIR_DOMAIN_XML_UPDATE_CPU + # All flags: libvirt.VIR_DOMAIN_XML_INACTIVE, + # libvirt.VIR_DOMAIN_XML_MIGRATABLE, libvirt.VIR_DOMAIN_XML_SECURE, + # libvirt.VIR_DOMAIN_XML_UPDATE_CPU dest_file = "%s-inactive.xml" % (domain.name()) dest_file = os.path.join(path, dest_file) if os.path.exists(dest_file): - raise Exception, "File %s exists!!" % (dest_file) + raise Exception("File %s exists!!" % (dest_file)) dest_fh = open(dest_file, "w") @@ -96,13 +101,14 @@ def dumpXML(domain, path): dest_file = os.path.join(path, dest_file) if os.path.exists(dest_file): - raise Exception, "File %s exists!!" % (dest_file) + raise Exception("File %s exists!!" % (dest_file)) dest_fh = open(dest_file, "w") # dump different xmls files. First of all, the offline dump xml = domain.XMLDesc( - flags=libvirt.VIR_DOMAIN_XML_INACTIVE+libvirt.VIR_DOMAIN_XML_MIGRATABLE) + flags=libvirt.VIR_DOMAIN_XML_INACTIVE + + libvirt.VIR_DOMAIN_XML_MIGRATABLE) dest_fh.write(xml) dest_fh.close() @@ -131,7 +137,11 @@ def getDisks(domain): # iterate amoung sources and targets if len(sources) != len(targets): - raise Exception, "Targets and sources lengths are different %s:%s" % (len(sources), len(targets)) + raise Exception( + "Targets and sources lengths are different %s:%s" % ( + len(sources), len(targets) + ) + ) # here all the devices I want to back up devs = {} @@ -237,7 +247,7 @@ def getSnapshotXML(self): (my_cmds, create_xml.stderr.read())) logger.critical("{exe} returned {stato} state".format( stato=status, exe=my_cmds[0])) - raise Exception, "snapshot-create-as didn't work properly" + raise Exception("snapshot-create-as didn't work properly") return self.snapshot_xml @@ -315,9 +325,10 @@ def doBlockCommit(self): (my_cmds, blockcommit.stderr.read())) logger.critical("{exe} returned {stato} state".format( stato=status, exe=my_cmds[0])) - raise Exception, "blockcommit didn't work properly" + raise Exception("blockcommit didn't work properly") - # After blockcommit, I need to check that image were successfully pivoted + # After blockcommit, I need to check that image were successfully + # pivoted test_disks = self.getDisks() for disk, base in self.disks.iteritems(): @@ -332,7 +343,8 @@ def doBlockCommit(self): else: logger.error("original base: %s, top: %s, new_base: %s" % (base, top, test_base)) - raise Exception, "Something goes wrong for snaphost %s" % (self.snapshotId) + raise Exception( + "Something goes wrong for snaphost %s" % (self.snapshotId)) # If I arrive here, I can delete snapshot self.__snapshotDelete() @@ -393,4 +405,4 @@ def packArchive(target): logger.error("Error for %s:%s" % (my_cmds, pigz.stderr.read())) logger.critical("{exe} returned {stato} state".format( stato=status, exe=my_cmds[0])) - raise Exception, "pigz didn't work properly" + raise Exception("pigz didn't work properly") diff --git a/kvmBackup.py b/kvmBackup.py index 53288e6..ca8e0e9 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -24,6 +24,8 @@ """ +from __future__ import print_function + import argparse import datetime import logging @@ -113,7 +115,7 @@ def backup(domain, parameters, backupdir): # check if no snapshot are defined if snapshot.hasCurrentSnapshot() is True: - raise Exception, "Domain '%s' has already a snapshot" % (domain) + raise Exception("Domain '%s' has already a snapshot" % (domain)) # changing directory olddir = os.getcwd() @@ -279,7 +281,7 @@ def backup(domain, parameters, backupdir): try: backup(domain_name, parameters, backupdir) - except Exception, message: + except Exception as message: logger.exception(message) logger.error("Domain '%s' was not backed up" % (domain_name)) From 50a0cf743037f113dd1c33367d0ac6d3c3c3d602 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Mon, 7 Feb 2022 13:03:53 +0100 Subject: [PATCH 05/13] :rotating_light: remove linter warnings --- Lib/flock.py | 12 +++++---- Lib/helper.py | 74 ++++++++++++++++++++++++++++++++++----------------- kvmBackup.py | 15 +++++++---- 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/Lib/flock.py b/Lib/flock.py index f4a07b2..6350772 100644 --- a/Lib/flock.py +++ b/Lib/flock.py @@ -72,11 +72,13 @@ def acquire(self): fh.close() if self.debug: logger.debug('Acquired lock: %s' % self.fddr()) - except: + + # TODO: catch the proper exception + except Exception: if os.path.isfile(self.path): try: os.unlink(self.path) - except: + except Exception: pass raise (self.FileLockAcquisitionError, 'Error acquiring lock: %s' % self.fddr()) @@ -89,7 +91,7 @@ def release(self): os.unlink(self.path) if self.debug: logger.debug('Released lock: %s' % self.fddr()) - except: + except Exception: raise (self.FileLockReleaseError, 'Error releasing lock: %s' % self.fddr()) return self @@ -103,7 +105,7 @@ def _readlock(self): fh.close() lock['pid'], lock['host'] = data return lock - except: + except Exception: return {'pid': 8**10, 'host': ''} def islocked(self): @@ -112,7 +114,7 @@ def islocked(self): lock = self._readlock() os.kill(int(lock['pid']), 0) return (lock['host'] == self.host) - except: + except Exception: return False def ownlock(self): diff --git a/Lib/helper.py b/Lib/helper.py index 008c421..9a005ba 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -46,8 +46,9 @@ # A global connection instance conn = libvirt.open("qemu:///system") -# una funzione che ho trovato qui: https://blog.nelhage.com/2010/02/a-very-subtle-bug/ -# e che dovrebbe gestire i segnali strani quando esco da un suprocess +# a function found here: +# https://blog.nelhage.com/2010/02/a-very-subtle-bug/ +# which attempt to deal with signals when exiting subprocess def preexec_fn(): return signal.signal(signal.SIGPIPE, signal.SIG_DFL) @@ -126,7 +127,8 @@ def getDisks(domain): # the fromstring method returns the root node root = ET.fromstring(domain.XMLDesc()) - # then use XPath to search a line like under tag + # then use XPath to search a line like + # under tag devices = root.findall("./devices/disk[@device='disk']") # Now find the child element with source tag @@ -205,8 +207,8 @@ def hasCurrentSnapshot(self): return True def getSnapshotXML(self): - """Since I need to do a Snapshot with a XML file, I will create an XML to call - the appropriate libvirt method""" + """Since I need to do a Snapshot with a XML file, I will create an XML + to call the appropriate libvirt method""" # get my domain domain = self.getDomain() @@ -221,11 +223,17 @@ def getSnapshotXML(self): diskspecs = [] for disk in self.disks.iterkeys(): - diskspecs += ["--diskspec %s,file=/var/lib/libvirt/images/snapshot_%s_%s-%s.img" % - (disk, self.domain_name, disk, self.snapshotId)] - - my_cmd = "virsh snapshot-create-as --domain {domain_name} {snapshotId} {diskspecs} --disk-only --atomic --quiesce --print-xml".format( - domain_name=domain.name(), snapshotId=self.snapshotId, diskspecs=" ".join(diskspecs)) + diskspecs += [ + "--diskspec %s,file=/var/lib/libvirt/images/snapshot" + "_%s_%s-%s.img" % ( + disk, self.domain_name, disk, self.snapshotId)] + + my_cmd = ( + "virsh snapshot-create-as --domain {domain_name} {snapshotId} " + "{diskspecs} --disk-only --atomic --quiesce --print-xml").format( + domain_name=domain.name(), + snapshotId=self.snapshotId, + diskspecs=" ".join(diskspecs)) logger.debug("Executing: %s" % (my_cmd)) @@ -234,7 +242,10 @@ def getSnapshotXML(self): # Launch command create_xml = subprocess.Popen( - my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) + my_cmds, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec_fn, + shell=False) # read output in xml self.snapshot_xml = create_xml.stdout.read() @@ -265,8 +276,10 @@ def callSnapshot(self): self.getSnapshotXML() # Those are the flags I need for creating SnapShot - [disk_only, atomic, quiesce] = [libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY, - libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_ATOMIC, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE] + [disk_only, atomic, quiesce] = [ + libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY, + libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_ATOMIC, + libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE] # get a domain domain = self.getDomain() @@ -277,13 +290,15 @@ def callSnapshot(self): self.snapshot = domain.snapshotCreateXML( self.snapshot_xml, flags=sum([disk_only, atomic, quiesce])) - # Once i've created a snapshot, I can read disks to have snapshot image name + # Once i've created a snapshot, I can read disks to have snapshot + # image name self.snapshot_disk = self.getDisks() # debug for disk, top in self.snapshot_disk.iteritems(): - logger.debug("Created top image {top} for {domain_name} {disk}".format( - top=top, domain_name=domain.name(), disk=disk)) + logger.debug( + "Created top image {top} for {domain_name} {disk}".format( + top=top, domain_name=domain.name(), disk=disk)) return self.snapshot @@ -295,11 +310,14 @@ def doBlockCommit(self): logger.info("Blockcommitting %s" % (domain.name())) - # A blockcommit for every disks. Using names like libvirt variables. Base is the original image file + # A blockcommit for every disks. Using names like libvirt variables. + # Base is the original image file for disk in self.disks.iterkeys(): # the command to execute - my_cmd = "virsh blockcommit {domain_name} {disk} --active --verbose --pivot".format( - domain_name=domain.name(), disk=disk) + my_cmd = ( + "virsh blockcommit {domain_name} {disk} --active " + "--verbose --pivot").format( + domain_name=domain.name(), disk=disk) logger.debug("Executing: %s" % (my_cmd)) # split the executable @@ -307,12 +325,16 @@ def doBlockCommit(self): # Launch command blockcommit = subprocess.Popen( - my_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) + my_cmds, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec_fn, + shell=False) # read output throug processing for line in blockcommit.stdout: line = line.strip() - if len(line) is 0: + if len(line) == 0: continue logger.debug("%s" % (line)) @@ -387,13 +409,17 @@ def packArchive(target): my_cmds = shlex.split(my_cmd) # Launch command - pigz = subprocess.Popen(my_cmds, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, preexec_fn=preexec_fn, shell=False) + pigz = subprocess.Popen( + my_cmds, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec_fn, + shell=False) # read output throug processing for line in pigz.stdout: line = line.strip() - if len(line) is 0: + if len(line) == 0: continue logger.debug("%s" % (line)) diff --git a/kvmBackup.py b/kvmBackup.py index ca8e0e9..10143f2 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -46,12 +46,14 @@ # Logging istance logging.basicConfig( - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + level=logging.DEBUG) logger = logging.getLogger(prog_name) notice = """ kvmBackup.py Copyright (C) 2015-2016 PTP -This program comes with ABSOLUTELY NO WARRANTY; for details type `kvmBackup.py --help'. +This program comes with ABSOLUTELY NO WARRANTY; for details type + `kvmBackup.py --help'. This is free software, and you are welcome to redistribute it under certain conditions; see LICENSE.txt for details. @@ -220,8 +222,10 @@ def backup(domain, parameters, backupdir): type=str, help="The config file") parser.add_argument("--force", required=False, action='store_true', default=False, help="Force backup (with rotation)") - parser.add_argument("--domains", required=False, type=str, - help="comma separated list of domains to backup ('virsh list --all' to get domains)") + parser.add_argument( + "--domains", required=False, type=str, + help=("comma separated list of domains to backup ('virsh list " + "--all' to get domains)")) args = parser.parse_args() # logging notice @@ -241,7 +245,8 @@ def backup(domain, parameters, backupdir): if not lock: logger.error( - "Another istance of '%s' is running. Please wait for its termination or kill the running application" % (sys.argv[0])) + "Another istance of '%s' is running. Please wait for its " + "termination or kill the running application" % (sys.argv[0])) sys.exit(-1) # get all domain names From 48581fae60344d0a280cf27f9faec90efb294c23 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Mon, 7 Feb 2022 13:11:12 +0100 Subject: [PATCH 06/13] :bug: fix relative imports --- Lib/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/__init__.py b/Lib/__init__.py index 4e41f4a..daf4d22 100644 --- a/Lib/__init__.py +++ b/Lib/__init__.py @@ -22,8 +22,8 @@ @author: Paolo Cozzi """ -import helper -import flock +from . import helper +from . import flock __author__ = "Paolo Cozzi" __version__ = "1.1" From c987299af60dc78f1e829de498e0702c95053821 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Mon, 7 Feb 2022 13:30:13 +0100 Subject: [PATCH 07/13] :bug: replace d.iteritems with iter(d.items()) --- Lib/helper.py | 4 ++-- kvmBackup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/helper.py b/Lib/helper.py index 9a005ba..a535276 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -295,7 +295,7 @@ def callSnapshot(self): self.snapshot_disk = self.getDisks() # debug - for disk, top in self.snapshot_disk.iteritems(): + for disk, top in iter(self.snapshot_disk.items()): logger.debug( "Created top image {top} for {domain_name} {disk}".format( top=top, domain_name=domain.name(), disk=disk)) @@ -353,7 +353,7 @@ def doBlockCommit(self): # pivoted test_disks = self.getDisks() - for disk, base in self.disks.iteritems(): + for disk, base in iter(self.disks.items()): test_base = test_disks[disk] top = self.snapshot_disk[disk] diff --git a/kvmBackup.py b/kvmBackup.py index 10143f2..909fb9a 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -178,7 +178,7 @@ def backup(domain, parameters, backupdir): (domain, tar_path)) # copying file - for disk, source in snapshot.disks.iteritems(): + for disk, source in iter(snapshot.disks.items()): dest = os.path.join(datadir, os.path.basename(source)) logger.debug("copying '%s' to '%s'" % (source, dest)) @@ -268,7 +268,7 @@ def backup(domain, parameters, backupdir): # debug # pprint.pprint(mydomains) - for domain_name, parameters in mydomains.iteritems(): + for domain_name, parameters in iter(mydomains.items()): # check if bakcup is needed domain_backup = False From b6c9727f7cd0dc68bf8019833e947ed9b52fbc86 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Mon, 7 Feb 2022 15:42:50 +0100 Subject: [PATCH 08/13] :bug: fix other issues related to python2 to 3 migration --- Lib/helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/helper.py b/Lib/helper.py index a535276..342427f 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -222,7 +222,7 @@ def getSnapshotXML(self): # now construct all diskspec diskspecs = [] - for disk in self.disks.iterkeys(): + for disk in iter(self.disks): diskspecs += [ "--diskspec %s,file=/var/lib/libvirt/images/snapshot" "_%s_%s-%s.img" % ( @@ -248,7 +248,7 @@ def getSnapshotXML(self): shell=False) # read output in xml - self.snapshot_xml = create_xml.stdout.read() + self.snapshot_xml = create_xml.stdout.read().decode("ascii") # Lancio il comando e aspetto che termini status = create_xml.wait() @@ -312,7 +312,7 @@ def doBlockCommit(self): # A blockcommit for every disks. Using names like libvirt variables. # Base is the original image file - for disk in self.disks.iterkeys(): + for disk in iter(self.disks): # the command to execute my_cmd = ( "virsh blockcommit {domain_name} {disk} --active " From d3c8afdc4095a572180778843df9c063ce51f647 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Mon, 7 Feb 2022 15:43:23 +0100 Subject: [PATCH 09/13] :sparkles: deal with inactive domain skip inactive domains from backup and raise exceptions --- Lib/helper.py | 46 ++++++++++++++++++++-------------------------- kvmBackup.py | 14 ++++++++++---- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Lib/helper.py b/Lib/helper.py index 342427f..bd23019 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -176,31 +176,34 @@ def getDomain(self): return self.conn.lookupByName(self.domain_name) + @property + def domain(self): + return self.getDomain() + + def domainIsActive(self): + """Return true if domain is active (VM up and running)""" + + if self.domain.isActive(): + return True + + return False + def getDisks(self): """Call getDisk on my instance""" - # get my domain - domain = self.getDomain() - # call getDisk to get the disks to do snapshot - return getDisks(domain) + return getDisks(self.domain) def dumpXML(self, path): """Call dumpXML on my instance""" - # get my domain - domain = self.getDomain() - # call getDisk to get the disks to do snapshot - return dumpXML(domain, path) + return dumpXML(self.domain, path) def hasCurrentSnapshot(self): """call hasCurrentSnapshot on domain class attribute""" - # get my domain - domain = self.getDomain() - - if domain.hasCurrentSnapshot() == 0: + if self.domain.hasCurrentSnapshot() == 0: return False else: @@ -210,9 +213,6 @@ def getSnapshotXML(self): """Since I need to do a Snapshot with a XML file, I will create an XML to call the appropriate libvirt method""" - # get my domain - domain = self.getDomain() - # call getDisk to get the disks to do snapshot self.disks = self.getDisks() @@ -231,7 +231,7 @@ def getSnapshotXML(self): my_cmd = ( "virsh snapshot-create-as --domain {domain_name} {snapshotId} " "{diskspecs} --disk-only --atomic --quiesce --print-xml").format( - domain_name=domain.name(), + domain_name=self.domain.name(), snapshotId=self.snapshotId, diskspecs=" ".join(diskspecs)) @@ -281,13 +281,10 @@ def callSnapshot(self): libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_ATOMIC, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE] - # get a domain - domain = self.getDomain() - # do a snapshot logger.info("Creating snapshot %s for %s" % (self.snapshotId, self.domain_name)) - self.snapshot = domain.snapshotCreateXML( + self.snapshot = self.domain.snapshotCreateXML( self.snapshot_xml, flags=sum([disk_only, atomic, quiesce])) # Once i've created a snapshot, I can read disks to have snapshot @@ -298,17 +295,14 @@ def callSnapshot(self): for disk, top in iter(self.snapshot_disk.items()): logger.debug( "Created top image {top} for {domain_name} {disk}".format( - top=top, domain_name=domain.name(), disk=disk)) + top=top, domain_name=self.domain.name(), disk=disk)) return self.snapshot def doBlockCommit(self): """Do a blockcommit for every disks shapshotted""" - # get a domain - domain = self.getDomain() - - logger.info("Blockcommitting %s" % (domain.name())) + logger.info("Blockcommitting %s" % (self.domain.name())) # A blockcommit for every disks. Using names like libvirt variables. # Base is the original image file @@ -317,7 +311,7 @@ def doBlockCommit(self): my_cmd = ( "virsh blockcommit {domain_name} {disk} --active " "--verbose --pivot").format( - domain_name=domain.name(), disk=disk) + domain_name=self.domain.name(), disk=disk) logger.debug("Executing: %s" % (my_cmd)) # split the executable diff --git a/kvmBackup.py b/kvmBackup.py index 909fb9a..7862fcb 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -59,10 +59,10 @@ """ -# A function to open a config file - def loadConf(file_conf): + """A function to open a config file""" + config = yaml.load(open(file_conf)) # read my defined domains @@ -77,10 +77,10 @@ def loadConf(file_conf): return mydomains, backupdir, config -# a function to check current day of the week - def checkDay(day): + """A function to check current day of the week""" + now = datetime.datetime.now() today = now.strftime("%a") @@ -115,6 +115,12 @@ def backup(domain, parameters, backupdir): # create a snapshot instance snapshot = helper.Snapshot(domain) + # check if domain is active + if not snapshot.domainIsActive(): + logger.error( + "domain '%s' is not Active: is VM up and running?" % domain) + raise NotImplementedError("Cannot backup an inactive domain!") + # check if no snapshot are defined if snapshot.hasCurrentSnapshot() is True: raise Exception("Domain '%s' has already a snapshot" % (domain)) From 7b82a0434be1cfa1d58325ea1f4c3149255a6ab9 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Tue, 8 Feb 2022 15:34:52 +0100 Subject: [PATCH 10/13] :sparkles: check QEMU guest agent during backup --- Lib/helper.py | 23 +++++++++++++++++++++++ TODO | 2 +- kvmBackup.py | 7 +++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/Lib/helper.py b/Lib/helper.py index bd23019..ca01933 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -34,11 +34,13 @@ import signal import subprocess import uuid +import json # To inspect xml import xml.etree.ElementTree as ET import libvirt +import libvirt_qemu # Logging istance logger = logging.getLogger(__name__) @@ -188,6 +190,27 @@ def domainIsActive(self): return False + def domainHasGuestAgent(self): + """Test if Guest Agent is up and running""" + + try: + response = libvirt_qemu.qemuAgentCommand( + self.domain, + '{"execute":"guest-ping"}', + timeout=30, + flags=0) + + except libvirt.libvirtError as error: + logger.error(error) + return False + + data = json.loads(response) + + if 'return' in data: + return True + + return False + def getDisks(self): """Call getDisk on my instance""" diff --git a/TODO b/TODO index e59622e..b2b42b2 100644 --- a/TODO +++ b/TODO @@ -5,6 +5,6 @@ * copy CD-rom data * testing wrong configurations * rotating using dates, not numbers -* dealing with power off VM (delete snapshot will not update domain) +* dealing with power off VM * deal with empty configurations * python packaging? diff --git a/kvmBackup.py b/kvmBackup.py index 7862fcb..8943721 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -121,6 +121,13 @@ def backup(domain, parameters, backupdir): "domain '%s' is not Active: is VM up and running?" % domain) raise NotImplementedError("Cannot backup an inactive domain!") + # check that guest agent is Up and running + if not snapshot.domainHasGuestAgent(): + logger.error("QEMU guest agent is a requisite for a safe snapshot") + logger.error("Please check kvmBackup wiki pages for more info") + raise RuntimeError( + "Guest agent is not running. Check '%s' domain" % domain) + # check if no snapshot are defined if snapshot.hasCurrentSnapshot() is True: raise Exception("Domain '%s' has already a snapshot" % (domain)) From efc53889dd67912cb12068745b8f712e604b105b Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Tue, 8 Feb 2022 17:09:33 +0100 Subject: [PATCH 11/13] :sparkles: deal with wrong configuration files and add verbosity --- TODO | 3 --- kvmBackup.py | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/TODO b/TODO index b2b42b2..51cb0d9 100644 --- a/TODO +++ b/TODO @@ -1,10 +1,7 @@ * e-mail notify for problems * add documentation -* verbosity level * copy CD-rom data -* testing wrong configurations * rotating using dates, not numbers * dealing with power off VM -* deal with empty configurations * python packaging? diff --git a/kvmBackup.py b/kvmBackup.py index 8943721..0bdc9a0 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -47,7 +47,7 @@ # Logging istance logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - level=logging.DEBUG) + level=logging.INFO) logger = logging.getLogger(prog_name) notice = """ @@ -65,15 +65,39 @@ def loadConf(file_conf): config = yaml.load(open(file_conf)) + if not config: + raise RuntimeError( + "Error in configuration file. Check for kvmbackup documentation") + # read my defined domains hostname = socket.gethostname() hostname = hostname.split(".")[0] # try to parse useful data - mydomains = config[hostname]["domains"] + mydomains = {} + backupdir = None + + if hostname not in config: + raise RuntimeError("Can't find '%s' in configuration file" % hostname) + + try: + mydomains = config[hostname]["domains"] + + except (TypeError, KeyError): + logger.warning("Cannot read 'domains' for '%s'" % hostname) + logger.warning("Is configuration file defined properly?") # get my backup directory - backupdir = config[hostname]["backupdir"] + try: + backupdir = config[hostname]["backupdir"] + + except (TypeError, KeyError): + logger.warning("Cannot read 'backupdir' for '%s'" % hostname) + logger.warning("Is configuration file defined properly?") + + if not mydomains or not backupdir: + raise RuntimeError( + "Error in configuration file. Check for kvmbackup documentation") return mydomains, backupdir, config @@ -229,6 +253,7 @@ def backup(domain, parameters, backupdir): # A global connection instance conn = libvirt.open("qemu:///system") + if __name__ == "__main__": parser = argparse.ArgumentParser(description='Backup of KVM-qcow2 domains') parser.add_argument("-c", "--config", required=True, @@ -239,8 +264,14 @@ def backup(domain, parameters, backupdir): "--domains", required=False, type=str, help=("comma separated list of domains to backup ('virsh list " "--all' to get domains)")) + parser.add_argument( + "-v", "--verbose", action='store_true', + help="verbose logging") args = parser.parse_args() + if args.verbose: + logger.setLevel(logging.DEBUG) + # logging notice sys.stderr.write(notice) sys.stderr.flush() @@ -287,7 +318,7 @@ def backup(domain, parameters, backupdir): # check if configuration domain exists or was filtered out if domain_name not in domains: - logger.warn("Ignoring domain '%s'" % (domain_name)) + logger.info("Ignoring domain '%s'" % (domain_name)) continue for day in parameters["day_of_week"]: @@ -309,7 +340,7 @@ def backup(domain, parameters, backupdir): break if domain_backup is False: - logger.debug("Ignoring '%s' domain" % (domain_name)) + logger.info("Ignoring '%s' domain" % (domain_name)) # end of the program if flag_errors is False: From fba805864adec46458ebfb2bbe165d656a856d3d Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Tue, 8 Feb 2022 17:15:30 +0100 Subject: [PATCH 12/13] :sparkles: limit cpus while compressing --- Lib/helper.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/helper.py b/Lib/helper.py index ca01933..9840581 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -27,14 +27,15 @@ from __future__ import print_function +import json import logging +import multiprocessing import os import shlex import shutil import signal import subprocess import uuid -import json # To inspect xml import xml.etree.ElementTree as ET @@ -416,10 +417,16 @@ def rotate(target, retention=3): shutil.move(target, target + '.1') -def packArchive(target): +def packArchive(target, cpu_limit=8): """Launch pigz for compressing files""" - my_cmd = "pigz --best --processes 8 %s" % (target) + cpus = multiprocessing.cpu_count() + + # setting number of cpus + if cpus > cpu_limit: + cpus = cpu_limit + + my_cmd = "pigz --best --processes %s %s" % (cpus, target) logger.debug("Executing: %s" % (my_cmd)) # split the executable From 64ec1b86383957f70c954896a5e9c7fe9c2664e8 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Tue, 8 Feb 2022 17:28:40 +0100 Subject: [PATCH 13/13] :page_facing_up: update license --- Lib/__init__.py | 4 ++-- Lib/flock.py | 4 ++-- Lib/helper.py | 4 ++-- README.md | 2 +- config.yml | 2 +- kvmBackup.py | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/__init__.py b/Lib/__init__.py index daf4d22..943faea 100644 --- a/Lib/__init__.py +++ b/Lib/__init__.py @@ -2,7 +2,7 @@ """ kvmBackup - a software for snapshotting KVM images and backing them up -Copyright (C) 2015-2016 PTP +Copyright (C) 2015-2022 Paolo Cozzi This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Created on Wed Oct 21 13:56:55 2015 -@author: Paolo Cozzi +@author: Paolo Cozzi """ from . import helper diff --git a/Lib/flock.py b/Lib/flock.py index 6350772..2c897eb 100644 --- a/Lib/flock.py +++ b/Lib/flock.py @@ -2,7 +2,7 @@ """ kvmBackup - a software for snapshotting KVM images and backing them up -Copyright (C) 2015-2016 PTP +Copyright (C) 2015-2022 Paolo Cozzi This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Created on Wed Oct 21 13:08:56 2015 -@author: Paolo Cozzi +@author: Paolo Cozzi Simple lockfile to detect previous instances of app (Python recipe http://code.activestate.com/recipes/498171/) diff --git a/Lib/helper.py b/Lib/helper.py index 9840581..5ada3bf 100644 --- a/Lib/helper.py +++ b/Lib/helper.py @@ -2,7 +2,7 @@ """ kvmBackup - a software for snapshotting KVM images and backing them up -Copyright (C) 2015-2016 PTP +Copyright (C) 2015-2022 Paolo Cozzi This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Created on Tue Oct 20 12:43:39 2015 -@author: Paolo Cozzi +@author: Paolo Cozzi A module to deal with KVM backup diff --git a/README.md b/README.md index f854355..ef83a8a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ found in our [Wiki](https://github.com/bioinformatics-ptp/kvmBackup/wiki) ## License kvmBackup - a software for snapshotting KVM images and backing them up -Copyright (C) 2015-2016 PTP +Copyright (C) 2015-2022 Paolo Cozzi This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/config.yml b/config.yml index d2a7938..41e04ed 100644 --- a/config.yml +++ b/config.yml @@ -1,6 +1,6 @@ # # kvmBackup - a software for snapshotting KVM images and backing them up -# Copyright (C) 2015-2016 PTP +# Copyright (C) 2015-2022 Paolo Cozzi # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/kvmBackup.py b/kvmBackup.py index 0bdc9a0..f856d15 100755 --- a/kvmBackup.py +++ b/kvmBackup.py @@ -3,7 +3,7 @@ """ kvmBackup - a software for snapshotting KVM images and backing them up -Copyright (C) 2015-2016 PTP +Copyright (C) 2015-2022 Paolo Cozzi This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -20,7 +20,7 @@ Created on Fri Oct 9 11:20:46 2015 -@author: Paolo Cozzi +@author: Paolo Cozzi """ @@ -51,7 +51,7 @@ logger = logging.getLogger(prog_name) notice = """ -kvmBackup.py Copyright (C) 2015-2016 PTP +kvmBackup.py Copyright (C) 2015-2022 Paolo Cozzi This program comes with ABSOLUTELY NO WARRANTY; for details type `kvmBackup.py --help'. This is free software, and you are welcome to redistribute it