]>
git.sthu.org Git - sitarba.git/blob - sitarba
2 """A simple backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Modes
= ["full", "incr", "diff"]
26 "hour" : datetime
.timedelta(0, 3600),
27 "day" : datetime
.timedelta(1),
28 "week" : datetime
.timedelta(7),
29 "month" : datetime
.timedelta(31),
30 "year" : datetime
.timedelta(365) }
32 def __init__(self
, unit
=None, mult
=1, mode
="full", numkeeps
=None):
36 self
.numkeeps
= numkeeps
40 return "[unit: " + repr(self
.unit
) + \
41 ", mult:" + repr(self
.mult
) + \
42 ", mode: " + repr(self
.mode
) + \
43 ", numkeeps: " + repr(self
.numkeeps
) + \
44 ", excludes: " + repr(self
.excludes
) + "]"
46 def getTimeDelta(self
):
49 return self
.mult
*Epoch
.units
[self
.unit
]
51 def isRipe(self
, oldest
, now
):
59 if delta
>= self
.getTimeDelta():
62 if self
.unit
== "hour":
63 return abs(now
.hour
- oldest
.hour
) >= mult
64 elif self
.unit
== "day":
65 return abs(now
.day
- oldest
.day
) >= mult
66 elif self
.unit
== "week":
67 return abs(now
.isocalendar()[1] - oldest
.isocalendar()[1]) >= mult
68 elif self
.unit
== "month":
69 return abs(now
.month
- oldest
.month
) >= mult
70 elif self
.unit
== "year":
71 return abs(now
.year
- oldest
.year
) >= mult
77 def parseTimedelta( deltastr
):
78 tokens
= [ s
.strip() for s
in deltastr
.split("*") ]
83 elif len(tokens
) == 2:
87 raise ValueError("Invalid format: '{0}'".format(deltastr
))
89 if not unit
in Epoch
.units
:
90 raise ValueError("Unknown unit '{0}'".format(unit
))
93 raise ValueError("Non-positive factor '{0}' given.".format(mult
))
100 """A fileset has a name and a list of directories."""
101 def __init__(self
, name
, dirs
, excludes
):
104 self
.excludes
= excludes
107 return "[name: " + self
.name
+ \
108 ", dirs: " + str(self
.dirs
) + \
109 ", excludes: " + str(self
.excludes
) + "]"
113 """A single backup has a date, an epoch and a mode."""
115 def __init__(self
, date
, epoch
, mode
):
122 def fromDirName(dirname
):
123 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
125 if not mode
in Modes
:
126 raise ValueError("Invalid mode: " + mode
)
128 date
= datetime
.datetime(int(strdate
[0:4]),
129 int(strdate
[4:6]), int(strdate
[6:8]),\
130 int(strtime
[0:2]), int(strtime
[2:4]))
132 return Backup(date
, epoch
, mode
)
135 return "[date: " + self
.date
.ctime() + \
136 ", epoch: " + self
.epoch
+ \
137 ", mode: " + self
.mode
+ "]"
139 def colAlignedString(self
):
140 age
= datetime
.datetime
.now() - self
.date
141 total_hours
= age
.total_seconds()/3600
142 if total_hours
<= 48:
143 agestr
= "(%s h)" % int(total_hours
)
145 agestr
= "(%s d)" % age
.days
146 return "%16s %9s %10s %4s" % (
147 self
.date
.strftime("%Y-%m-%d %H:%M"), agestr
,
148 self
.epoch
, self
.mode
)
151 def getDirName(date
, epoch
, mode
):
152 """Get directory name of backup by given properties."""
153 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
156 def isBackupDir(dirname
):
157 """Is directory a backup directory?"""
158 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
159 return p
.match(dirname
)
164 """Encapsules the configuration for the backup program."""
166 class ReadError(RuntimeError):
167 """An exception raised when reading configurations."""
168 def __init__(self
, value
):
173 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
175 # Filename where checksum of config is saved
176 checksumfn
= "checksum"
179 self
.backupdir
= None
180 self
.format
= self
.formats
[1]
181 self
.tarbin
= "/bin/tar"
185 self
.lastchecksum
= None
186 self
.epochs
= { "sporadic" : Epoch() }
190 return "[backupdir: " + self
.backupdir
+ \
191 ", format: " + self
.format
+ \
192 ", tarbin: " + self
.tarbin
+ \
193 ", excludes: " + repr(self
.excludes
) + \
194 ", epochs: " + repr(self
.epochs
) + \
195 ", sets: " + repr(self
.sets
) + "]"
197 def getRealEpochsSorted(self
):
198 """Return all epochs with have a non-None unit, sorted by
199 Epoch.getTimeDelta(), starting with the longest dela."""
201 realepochs
= [ e
for e
in epochs
.keys() if epochs
[e
].unit
!= None ]
202 deltakey
= lambda e
: epochs
[e
].getTimeDelta()
203 realepochs
.sort(key
=deltakey
, reverse
=True)
207 def _read_global(self
, config
, sec
):
208 for opt
in config
.options(sec
):
209 if opt
== "backupdir":
210 self
.backupdir
= config
.get(sec
, opt
)
211 if not os
.path
.isdir(self
.backupdir
):
212 raise Config
.ReadError("Backupdir '{0}' does not exist.".format(self
.backupdir
))
213 elif opt
== "format":
214 self
.format
= config
.get(sec
, opt
)
215 if not self
.format
in Config
.formats
:
216 raise Config
.ReadError("Invalid 'format' given.")
217 elif opt
== "tarbin":
218 self
.tarbin
= config
.get(sec
, opt
)
219 if not os
.path
.isfile(self
.tarbin
):
220 raise Config
.ReadError("Tar binary '{0}' does not exist.".format(self
.tarbin
))
221 elif opt
.startswith("exclude"):
222 self
.excludes
+= [ config
.get(sec
, opt
) ]
224 raise Config
.ReadError("Unknown option '{0}'.".format(opt
))
227 def _read_epoch(self
, config
, sec
):
228 name
= sec
[6:].strip()
230 if name
in self
.epochs
:
231 raise Config
.ReadError("Epoch '{0}' already defined.".format(name
))
232 p
= re
.compile(r
'^\w+$')
233 if not p
.match(name
):
234 raise Config
.ReadError("Epoch name '{0}' does not only " + \
235 "comprise alphanumeric characters.".format(name
))
236 if name
in Epoch
.units
:
239 for opt
in config
.options(sec
):
240 if opt
== "numkeeps":
242 e
.numkeeps
= int(config
.getint(sec
, opt
))
244 raise Config
.ReadError("Invalid integer given for '{0}'.".format(opt
))
246 raise Config
.ReadError("Non-positive numkeeps '{0}' given.".format(e
.numkeeps
))
249 e
.mode
= config
.get(sec
, opt
)
250 if not e
.mode
in Modes
:
251 raise Config
.ReadError("Invalid mode '{0}'.".format(e
.mode
))
253 elif opt
== "timespan":
254 if name
in Epoch
.units
:
255 raise Config
.ReadError("The time delta of a standard epoch " + \
256 "is not supposed to be redefined. ")
257 td
= config
.get(sec
, opt
)
259 mult
, unit
= Epoch
.parseTimedelta(td
)
262 except ValueError as e
:
263 raise Config
.ReadError("Invalid timespan '{0}': {1}".format(td
, str(e
)))
265 elif opt
.startswith("exclude"):
266 e
.excludes
+= [config
.get(sec
, opt
)]
269 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
271 if e
.numkeeps
== None:
272 raise Config
.ReadError("No numkeeps set for epoch '{0}'.".format(name
))
274 self
.epochs
[name
] = e
277 def _read_set(self
, config
, sec
):
278 name
= sec
[4:].strip()
279 p
= re
.compile(r
'^\w+$')
280 if not p
.match(name
):
281 raise Config
.ReadError("Set name '{0}' does not only " + \
282 "comprise alphanumeric characters.".format(name
))
287 for opt
in config
.options(sec
):
288 if opt
.startswith("dir"):
289 dirs
+= [config
.get(sec
, opt
)]
290 elif opt
.startswith("exclude"):
291 excludes
+= [config
.get(sec
, opt
)]
293 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
295 self
.sets
+= [FileSet(name
, dirs
, excludes
)]
298 def read(self
, filename
):
299 """Read configuration from file"""
301 if not os
.path
.isfile(filename
):
302 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
304 config
= configparser
.RawConfigParser()
305 config
.read(filename
)
307 for reqsec
in ["global"]:
308 if not config
.has_section(reqsec
):
309 raise Config
.ReadError("Mandatory section '" + reqsec
+ "' is missing.")
311 for sec
in config
.sections():
314 self
._read
_global
(config
, sec
)
316 elif sec
.startswith("epoch "):
317 self
._read
_epoch
(config
, sec
)
319 elif sec
.startswith("set "):
320 self
._read
_set
(config
, sec
)
323 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
325 if self
.backupdir
== None:
326 raise Config
.ReadError("No backup directory set.")
329 # Compute checksum of config file
331 f
= open(filename
, 'rb')
334 self
.checksum
= m
.hexdigest()
339 f
= open(os
.path
.join(self
.backupdir
, self
.checksumfn
), 'r')
340 self
.lastchecksum
= f
.read().strip()
343 self
.lastchecksum
= None
347 """List and create backups"""
349 def __init__(self
, conffn
):
351 self
.conf
.read(conffn
)
354 def listAllDirs(self
):
355 """List all dirs in backupdir"""
358 basedir
= self
.conf
.backupdir
359 dirs
= os
.listdir(basedir
)
361 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
364 def listExistingBackups(self
):
365 """Returns a list of old backups."""
369 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
370 backups
+= [ Backup
.fromDirName(entry
) ]
375 def getDesiredEpochs(self
, backups
, now
):
376 """Get desired epoch based on self.configuration and list of old backups"""
378 # Find the longest epoch for which we would like the make a backup
379 latest
= datetime
.datetime(1900, 1, 1)
380 for e
in self
.conf
.getRealEpochsSorted():
381 epoch
= self
.conf
.epochs
[e
]
382 if epoch
.numkeeps
<= 0:
385 # Get backups of that epoch
386 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
387 key
=lambda b
: b
.date
))
389 # If there are any, determine the latest
391 latest
= max(latest
, byepoch
[-1].date
)
393 if epoch
.isRipe(latest
, now
):
396 # No backup is to be made
401 def backupFileSet(self
, fileset
, targetdir
, excludes
, since
=None):
402 """Create an archive for given fileset at given target directory."""
404 logfile
= logging
.getLogger('backuplog')
405 logfile
.info("Running file set: " + fileset
.name
)
407 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
410 # Tar is verbose is sitarba is verbose
411 if LogConf
.con
.level
<= logging
.DEBUG
:
412 taropts
+= ["--verbose"]
414 # Add the since date, if given
416 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
418 # Add the exclude patterns
420 taropts
+= ["--exclude", pat
]
422 #Add exclude patterns from fileset
423 for pat
in fileset
.excludes
:
424 taropts
+= ["--exclude", pat
]
427 # Adding directories to backup
428 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
430 # Launch the tar process
431 tarargs
= [self
.conf
.tarbin
] + ["-cpaf", fsfn
] + taropts
432 logfile
.debug("tar call: " + " ".join(tarargs
))
433 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
434 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
436 # Change tarp's stdout and stderr to non-blocking
437 for s
in [tarp
.stdout
, tarp
.stderr
]:
439 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
440 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
442 # Read stdout and stderr of tarp
444 while tarp
.poll() == None:
445 rd
, wr
, ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
446 if tarp
.stdout
in rd
:
447 logging
.debug( tarp
.stdout
.readline()[:-1].decode() )
448 if tarp
.stderr
in rd
:
449 errmsg
+= tarp
.stderr
.read()
451 # Get the remainging output of tarp
452 for l
in tarp
.stdout
.readlines():
453 logging
.debug(l
.decode().rstrip())
454 errmsg
+= tarp
.stderr
.read()
456 # Get return code of tarp
459 for l
in errmsg
.decode().split("\n"):
461 logfile
.error(self
.conf
.tarbin
+ " returned with exit status " + \
465 def backup(self
, epoch
=None, mode
=None):
466 """Make a new backup, if necessary. If epoch is None then determine
467 desired epoch automatically. Use given epoch otherwise. If mode is None
468 then use mode for given epoch. Use given mode otherwise."""
470 now
= datetime
.datetime
.now()
471 oldbackups
= self
.listExistingBackups()
473 # Get epoch of backup
475 epoch
= self
.getDesiredEpochs(oldbackups
, now
)
477 logging
.info("No backup planned.")
482 mode
= self
.conf
.epochs
[epoch
].mode
483 logging
.info("Making a backup. Epochs: " + epoch
+ ", mode: " + mode
)
485 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
487 # No old full backups existing
488 if mode
!= "full" and len(oldfullbackups
) == 0:
489 logging
.info("No full backups existing. Making a full backup.")
491 # Checksum changed -> self.config file changed
492 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
493 logging
.warning("Full backup recommended as config file has changed.")
496 # If we have a full backup, we backup everything
499 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
501 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
504 logging
.debug("Making backup relative to " + since
.ctime())
509 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
513 # Create new backup directory
514 basedir
= self
.conf
.backupdir
515 dirname
= Backup
.getDirName(now
, epoch
, mode
)
516 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
517 targetdir
= os
.path
.join(basedir
, tmpdirname
)
522 logfile
= logging
.getLogger("backuplog")
523 fil
= logging
.FileHandler(os
.path
.join(targetdir
, "log"))
524 fil
.setLevel(logging
.DEBUG
)
525 logfile
.addHandler(fil
)
527 logfile
.info("Started: " + now
.ctime())
529 # Backup all file sets
530 for s
in self
.conf
.sets
:
531 excludes
= self
.conf
.excludes
+ self
.conf
.epochs
[epoch
].excludes
532 self
.backupFileSet(s
, targetdir
, excludes
, since
)
534 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
537 # Rename backup directory to final name
538 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
540 # We made a full backup -- recall checksum of config
542 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
543 f
.write( self
.conf
.checksum
)
549 """Prune old backup files"""
551 allDirs
= sorted(self
.listAllDirs())
552 # Collect all directories that are removed
553 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
556 backups
= self
.listExistingBackups()
557 # Group backups by epoch and sort them by age
558 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
559 key
=lambda b
: b
.date
, reverse
=True)) \
560 for e
in self
.conf
.getRealEpochsSorted() }
561 # If we have too many backups of a specific epoch --> add them to remove list
563 epoch
= self
.conf
.epochs
[e
]
564 old
= byepoch
[e
][epoch
.numkeeps
:]
565 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
568 logging
.info("List of stale/outdated entries:")
576 if Backup
.isBackupDir(d
):
577 msg
+= Backup
.fromDirName(d
).colAlignedString()
583 # Check that dirs to be removed is in list of all dirs
585 assert( d
in allDirs
)
587 if len(removeDirs
) == 0:
588 logging
.info("No stale/outdated entries to remove.")
594 basedir
= self
.conf
.backupdir
595 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
599 shutil
.rmtree(os
.path
.join(basedir
, d
))
601 logging
.error("Error when removing '%s': %s" % (d
, e
.strerror
) )
604 def ask_user_yesno(self
, question
):
605 if LogConf
.con
.level
<= logging
.INFO
:
606 return input(question
)
612 """Print --help text"""
614 print("sitarba - a simple backup solution.")
617 print(" " + sys
.argv
[0] + " {options} [cmd]")
618 print(" " + sys
.argv
[0] + " --help")
621 print(" backup make a new backup, if necessary")
622 print(" list list all backups (default)")
623 print(" prune prune outdated/old backups")
626 print(" -h, --help print this usage text")
627 print(" -c, --conf FILE use given configuration file")
628 print(" default: /etc/sitarba.conf")
629 print(" -e, --epoch EPOCH force to create backup for given epoch, which")
630 print(" can be 'sporadic' or one of the configured epochs")
631 print(" -m, --mode MODE override mode: full, diff, or incr")
632 print(" -n, --dry-run don't do anything, just tell what would be done")
633 print(" -v, --verbose be more verbose and interact with user")
634 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
635 print(" error, warning, info, debug")
636 print(" -V, --version print version info")
641 """Encapsulates logging configuration"""
643 con
= logging
.StreamHandler(sys
.stderr
)
647 """Setup logging system"""
648 conlog
= logging
.getLogger()
649 conlog
.setLevel(logging
.DEBUG
)
651 cls
.con
.setLevel(logging
.WARNING
)
652 conlog
.addHandler(cls
.con
)
654 fillog
= logging
.getLogger("backuplog")
655 fillog
.setLevel(logging
.DEBUG
)
658 if __name__
== "__main__":
662 conffn
= "/etc/sitarba.conf"
668 while i
< len(sys
.argv
)-1:
672 if opt
in ["-h", "--help"]:
676 elif opt
in ["-c", "--conf"]:
680 elif opt
in ["-V", "--version"]:
681 print("sitarba " + __version__
)
684 elif opt
in ["-v", "--verbose"]:
685 LogConf
.con
.setLevel(logging
.INFO
)
687 elif opt
in ["--verbosity"]:
690 numlevel
= getattr(logging
, level
.upper(), None)
691 if not isinstance(numlevel
, int):
692 raise ValueError('Invalid verbosity level: %s' % level
)
693 LogConf
.con
.setLevel(numlevel
)
695 elif opt
in ["-m", "--mode"]:
698 if not mode
in Modes
:
699 logging
.error("Unknown mode '" + mode
+ "'.")
702 elif opt
in ["-n", "--dry-run"]:
703 Options
.dryrun
= True
705 elif opt
in ["-e", "--epoch"]:
709 elif opt
in ["backup", "list", "prune"]:
713 logging
.error("Unknown option: " + opt
)
717 man
= BackupManager(conffn
)
719 logging
.debug("Config: " + str(man
.conf
))
721 if epoch
!= None and not epoch
in man
.conf
.epochs
.keys():
722 logging
.error("Unknown epoch '" + epoch
+ "'.")
726 man
.backup(epoch
, mode
)
729 for b
in sorted(man
.listExistingBackups(), key
=lambda b
: b
.date
):
730 print(b
.colAlignedString())
735 except (Config
.ReadError
, configparser
.Error
) as e
:
736 logging
.error("Error: " + e
.message
)