c236ba0589e68018b41991ba3231ef40b6396ed3
2 """A simple backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Modes
= ["full", "incr", "diff"]
21 "hour" : datetime
.timedelta(0, 3600),
22 "day" : datetime
.timedelta(1),
23 "week" : datetime
.timedelta(7),
24 "month" : datetime
.timedelta(31),
25 "year" : datetime
.timedelta(365) }
27 def __init__(self
, unit
=None, mult
=1, mode
="full", numkeeps
=None):
31 self
.numkeeps
= numkeeps
35 return "[unit: " + repr(self
.unit
) + \
36 ", mult:" + repr(self
.mult
) + \
37 ", mode: " + repr(self
.mode
) + \
38 ", numkeeps: " + repr(self
.numkeeps
) + \
39 ", excludes: " + repr(self
.excludes
) + "]"
41 def getTimeDelta(self
):
44 return self
.mult
*Epoch
.units
[self
.unit
]
46 def isRipe(self
, oldest
, now
):
54 if delta
>= self
.getTimeDelta():
57 if self
.unit
== "hour":
58 return abs(now
.hour
- oldest
.hour
) >= mult
59 elif self
.unit
== "day":
60 return abs(now
.day
- oldest
.day
) >= mult
61 elif self
.unit
== "week":
62 return abs(now
.isocalendar()[1] - oldest
.isocalendar()[1]) >= mult
63 elif self
.unit
== "month":
64 return abs(now
.month
- oldest
.month
) >= mult
65 elif self
.unit
== "year":
66 return abs(now
.year
- oldest
.year
) >= mult
72 def parseTimedelta( deltastr
):
73 tokens
= [ s
.strip() for s
in deltastr
.split("*") ]
78 elif len(tokens
) == 2:
82 raise ValueError("Invalid format: '{0}'".format(deltastr
))
84 if not unit
in Epoch
.units
:
85 raise ValueError("Unknown unit '{0}'".format(unit
))
88 raise ValueError("Non-positive factor '{0}' given.".format(mult
))
95 """A fileset has a name and a list of directories."""
96 def __init__(self
, name
, dirs
, excludes
):
99 self
.excludes
= excludes
102 return "[name: " + self
.name
+ \
103 ", dirs: " + str(self
.dirs
) + \
104 ", excludes: " + str(self
.excludes
) + "]"
108 """A single backup has a date, an epoch and a mode."""
110 def __init__(self
, date
, epoch
, mode
):
117 def fromDirName(dirname
):
118 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
120 if not mode
in Modes
:
121 raise ValueError("Invalid mode: " + mode
)
123 date
= datetime
.datetime(int(strdate
[0:4]),
124 int(strdate
[4:6]), int(strdate
[6:8]),\
125 int(strtime
[0:2]), int(strtime
[2:4]))
127 return Backup(date
, epoch
, mode
)
130 return "[date: " + self
.date
.ctime() + \
131 ", epoch: " + self
.epoch
+ \
132 ", mode: " + self
.mode
+ "]"
134 def colAlignedString(self
):
135 age
= datetime
.datetime
.now() - self
.date
136 total_hours
= age
.total_seconds()/3600
137 if total_hours
<= 48:
138 agestr
= "(%s h)" % int(total_hours
)
140 agestr
= "(%s d)" % age
.days
141 return "%16s %7s %10s %4s" % (
142 self
.date
.strftime("%Y-%m-%d %H:%M"), agestr
,
143 self
.epoch
, self
.mode
)
146 def getDirName(date
, epoch
, mode
):
147 """Get directory name of backup by given properties."""
148 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
151 def isBackupDir(dirname
):
152 """Is directory a backup directory?"""
153 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
154 return p
.match(dirname
)
159 """Encapsules the configuration for the backup program."""
161 class ReadError(RuntimeError):
162 """An exception raised when reading configurations."""
163 def __init__(self
, value
):
168 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
170 # Filename where checksum of config is saved
171 checksumfn
= "checksum"
174 self
.backupdir
= None
175 self
.format
= self
.formats
[1]
176 self
.tarbin
= "/bin/tar"
180 self
.lastchecksum
= None
181 self
.epochs
= Epochs
= { "sporadic" : Epoch() }
185 return "[backupdir: " + self
.backupdir
+ \
186 ", format: " + self
.format
+ \
187 ", tarbin: " + self
.tarbin
+ \
188 ", excludes: " + repr(self
.excludes
) + \
189 ", epochs: " + repr(self
.epochs
) + \
190 ", sets: " + repr(self
.sets
) + "]"
192 def getRealEpochsSorted(self
):
193 """Return all epochs with have a non-None unit, sorted by
194 Epoch.getTimeDelta(), starting with the longest dela."""
196 realepochs
= [ e
for e
in epochs
.keys() if epochs
[e
].unit
!= None ]
197 deltakey
= lambda e
: epochs
[e
].getTimeDelta()
198 realepochs
.sort(key
=deltakey
, reverse
=True)
202 def _read_global(self
, config
, sec
):
203 for opt
in config
.options(sec
):
205 self
.backupdir
= config
.get(sec
, opt
)
206 if not os
.path
.isdir(self
.backupdir
):
207 raise Config
.ReadError("Backupdir '{0}' does not exist.".format(self
.backupdir
))
209 self
.format
= config
.get(sec
, opt
)
210 if not self
.format
in Config
.formats
:
211 raise Config
.ReadError("Invalid 'format' given.")
213 self
.tarbin
= config
.get(sec
, opt
)
214 if not os
.path
.isfile(self
.tarbin
):
215 raise Config
.ReadError("Tar binary '{0}' does not exist.".format(self
.tarbin
))
216 elif opt
.startswith("exclude"):
217 self
.excludes
+= [ config
.get(sec
, opt
) ]
219 raise Config
.ReadError("Unknown option '{0}'.".format(opt
))
222 def _read_epoch(self
, config
, sec
):
223 name
= sec
[6:].strip()
225 if name
in self
.epochs
:
226 raise Config
.ReadError("Epoch '{0}' already defined.".format(name
))
227 if name
in Epoch
.units
:
230 for opt
in config
.options(sec
):
233 e
.numkeeps
= int(config
.getint(sec
, opt
))
235 raise Config
.ReadError("Invalid integer given for '{0}'.".format(opt
))
237 raise Config
.ReadError("Non-positive numkeeps '{0}' given.".format(e
.numkeeps
))
240 e
.mode
= config
.get(sec
, opt
)
241 if not e
.mode
in Modes
:
242 raise Config
.ReadError("Invalid mode '{0}'.".format(e
.mode
))
244 elif opt
=="timespan":
245 if name
in Epoch
.units
:
246 raise Config
.ReadError("The time delta of a standard epoch " + \
247 "is not supposed to be redefined. ")
248 td
= config
.get(sec
,opt
)
250 mult
, unit
= Epoch
.parseTimedelta(td
)
253 except ValueError as e
:
254 raise Config
.ReadError("Invalid timespan '{0}': {1}".format(td
, str(e
)))
256 elif opt
.startswith("exclude"):
257 e
.excludes
+= [config
.get(sec
, opt
)]
260 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
262 if e
.numkeeps
== None:
263 raise Config
.ReadError("No numkeeps set for epoch '{0}'.".format(name
))
265 self
.epochs
[name
] = e
268 def _read_set(self
, config
, sec
):
269 name
= sec
[4:].strip()
273 for opt
in config
.options(sec
):
274 if opt
.startswith("dir"):
275 dirs
+= [config
.get(sec
, opt
)]
276 elif opt
.startswith("exclude"):
277 excludes
+= [config
.get(sec
,opt
)]
279 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
281 self
.sets
+= [FileSet(name
, dirs
, excludes
)]
284 def read(self
, filename
):
285 """Read configuration from file"""
287 if not os
.path
.isfile(filename
):
288 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
290 config
= configparser
.RawConfigParser()
291 config
.read(filename
)
293 for reqsec
in ["global"]:
294 if not config
.has_section(reqsec
):
295 raise Config
.ReadError("Mandatory section '" + reqsec
+ "' is missing.")
297 for sec
in config
.sections():
300 self
._read
_global
(config
, sec
)
302 elif sec
.startswith("epoch "):
303 self
._read
_epoch
(config
, sec
)
305 elif sec
.startswith("set "):
306 self
._read
_set
(config
, sec
)
309 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
311 if self
.backupdir
== None:
312 raise Config
.ReadError("No backup directory set.")
315 # Compute checksum of config file
317 f
= open(filename
, 'rb')
320 self
.checksum
= m
.hexdigest()
325 f
= open(os
.path
.join(self
.backupdir
, self
.checksumfn
), 'r')
326 self
.lastchecksum
= f
.read().strip()
329 self
.lastchecksum
= None
333 """List and create backups"""
335 def __init__(self
, conffn
):
337 self
.conf
.read(conffn
)
340 def listAllDirs(self
):
341 """List all dirs in backupdir"""
344 basedir
= self
.conf
.backupdir
345 dirs
= os
.listdir(basedir
)
347 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
350 def listOldBackups(self
):
351 """Returns a list of old backups."""
355 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
356 backups
+= [ Backup
.fromDirName(entry
) ]
361 def getDesiredEpochs(self
, backups
, now
):
362 """Get desired epoch based on self.configuration and list of old backups"""
364 # Find the longest epoch for which we would like the make a backup
365 latest
= datetime
.datetime(1900, 1, 1)
366 for e
in self
.conf
.getRealEpochsSorted():
367 epoch
= self
.conf
.epochs
[e
]
368 if epoch
.numkeeps
<= 0:
371 # Get backups of that epoch
372 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
373 key
=lambda b
: b
.date
))
375 # If there are any, determine the latest
377 latest
= max(latest
, byepoch
[-1].date
)
379 if epoch
.isRipe(latest
, now
):
382 # No backup is to be made
387 def backupFileSet(self
, fileset
, targetdir
, excludes
, since
=None):
388 """Create an archive for given fileset at given target directory."""
390 logfile
= logging
.getLogger('backuplog')
391 logfile
.info("Running file set: " + fileset
.name
)
393 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
396 # Tar is verbose is sitarba is verbose
397 if LogConf
.con
.level
<= logging
.INFO
:
398 taropts
+= ["--verbose"]
400 # Add the since date, if given
402 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
404 # Add the exclude patterns
406 taropts
+= ["--exclude", pat
]
408 #Add exclude patterns from fileset
409 for pat
in fileset
.excludes
:
410 taropts
+= ["--exclude", pat
]
413 # Adding directories to backup
414 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
416 # Launch the tar process
417 tarargs
= [self
.conf
.tarbin
] + ["-cpaf", fsfn
] + taropts
418 logfile
.debug("tar call: " + " ".join(tarargs
))
419 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
420 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
422 # Change tarp's stdout and stderr to non-blocking
423 for s
in [tarp
.stdout
, tarp
.stderr
]:
425 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
426 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
428 # Read stdout and stderr of tarp
430 while tarp
.poll() == None:
431 rd
,wr
,ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
432 if tarp
.stdout
in rd
:
433 logging
.debug( tarp
.stdout
.readline()[:-1].decode() )
434 if tarp
.stderr
in rd
:
435 errmsg
+= tarp
.stderr
.read()
437 # Get the remainging output of tarp
438 for l
in tarp
.stdout
.readlines():
439 logging
.debug(l
.decode().rstrip())
440 errmsg
+= tarp
.stderr
.read()
442 # Get return code of tarp
445 for l
in errmsg
.decode().split("\n"):
447 logfile
.error(self
.conf
.tarbin
+ " returned with exit status " + \
451 def backup(self
, epoch
=None, mode
=None):
452 """Make a new backup, if necessary. If epoch is None then determine
453 desired epoch automatically. Use given epoch otherwise. If mode is None
454 then use mode for given epoch. Use given mode otherwise."""
456 now
= datetime
.datetime
.now()
457 oldbackups
= self
.listOldBackups()
459 # Get epoch of backup
461 epoch
= self
.getDesiredEpochs(oldbackups
, now
)
463 logging
.info("No backup planned.")
468 mode
= self
.conf
.epochs
[epoch
].mode
469 logging
.info("Making a backup. Epochs: " + epoch
+ ", mode: " + mode
)
471 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
473 # No old full backups existing
474 if mode
!= "full" and len(oldfullbackups
)==0:
475 logging
.info("No full backups existing. Making a full backup.")
477 # Checksum changed -> self.config file changed
478 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
479 logging
.warning("Full backup recommended as config file has changed.")
482 # If we have a full backup, we backup everything
485 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
487 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
490 logging
.debug("Making backup relative to " + since
.ctime())
492 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
496 # Create new backup directory
497 basedir
= self
.conf
.backupdir
498 dirname
= Backup
.getDirName(now
, epoch
, mode
)
499 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
500 targetdir
= os
.path
.join(basedir
, tmpdirname
)
501 os
.mkdir( targetdir
)
505 logfile
= logging
.getLogger("backuplog")
506 fil
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
507 fil
.setLevel(logging
.DEBUG
)
508 logfile
.addHandler(fil
)
510 logfile
.info("Started: " + now
.ctime())
512 # Backup all file sets
513 for s
in self
.conf
.sets
:
514 excludes
= self
.conf
.excludes
+ self
.conf
.epochs
[epoch
].excludes
515 self
.backupFileSet(s
, targetdir
, excludes
, since
)
517 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
519 # Rename backup directory to final name
520 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
522 # We made a full backup -- recall checksum of config
524 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
525 f
.write( self
.conf
.checksum
)
531 """Prune old backup files"""
533 allDirs
= sorted(self
.listAllDirs())
534 # Collect all directories not matching backup name
535 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
537 # Get all directories which are kept
538 backups
= self
.listOldBackups()
540 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
541 key
=lambda b
: b
.date
, reverse
=True)) for e
in self
.conf
.getRealEpochsSorted() }
543 epoch
= self
.conf
.epochs
[e
]
544 old
= byepoch
[e
][epoch
.numkeeps
:]
545 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
548 logging
.info("List of stale/outdated entries:")
556 if Backup
.isBackupDir(d
):
557 msg
+= Backup
.fromDirName(d
).colAlignedString()
563 # Check that dirs to be removed is in list of all dirs
565 assert( d
in allDirs
)
567 if len(removeDirs
) == 0:
568 logging
.info("No stale/outdated entries to remove.")
571 basedir
= self
.conf
.backupdir
572 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
576 shutil
.rmtree(os
.path
.join(basedir
, d
))
578 logging
.error("Error when removing '%s': %s" % (d
,e
.strerror
) )
581 def ask_user_yesno(self
, question
):
582 if LogConf
.con
.level
<= logging
.INFO
:
583 return input(question
)
589 """Print --help text"""
591 print("sitarba - a simple backup solution.")
594 print(" " + sys
.argv
[0] + " {options} [cmd]")
595 print(" " + sys
.argv
[0] + " --help")
598 print(" backup make a new backup, if necessary")
599 print(" list list all backups (default)")
600 print(" prune prune outdated/old backups")
603 print(" -h, --help print this usage text")
604 print(" -c, --conf FILE use given configuration file")
605 print(" default: /etc/sitarba.conf")
606 print(" -e, --epoch EPOCH force to create backup for given epoch, which")
607 print(" can be 'sporadic' or one of the configured epochs")
608 print(" -m, --mode MODE override mode: full, diff, or incr")
609 print(" -v, --verbose be more verbose and interact with user")
610 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
611 print(" error, warning, info, debug")
612 print(" -V, --version print version info")
617 """Encapsulates logging configuration"""
619 con
= logging
.StreamHandler(sys
.stderr
)
623 """Setup logging system"""
624 conlog
= logging
.getLogger()
625 conlog
.setLevel(logging
.DEBUG
)
627 cls
.con
.setLevel(logging
.WARNING
)
628 conlog
.addHandler(cls
.con
)
630 fillog
= logging
.getLogger("backuplog")
631 fillog
.setLevel(logging
.DEBUG
)
634 if __name__
== "__main__":
638 conffn
= "/etc/sitarba.conf"
644 while i
< len(sys
.argv
)-1:
648 if opt
in ["-h", "--help"]:
652 elif opt
in ["-c", "--conf"]:
656 elif opt
in ["-V", "--version"]:
657 print("sitarba " + __version__
)
660 elif opt
in ["-v", "--verbose"]:
661 LogConf
.con
.setLevel(logging
.INFO
)
663 elif opt
in ["--verbosity"]:
666 numlevel
= getattr(logging
, level
.upper(), None)
667 if not isinstance(numlevel
, int):
668 raise ValueError('Invalid verbosity level: %s' % level
)
669 LogConf
.con
.setLevel(numlevel
)
671 elif opt
in ["-m", "--mode"]:
674 if not mode
in Modes
:
675 logging
.error("Unknown mode '" + mode
+ "'.")
678 elif opt
in ["-e", "--epoch"]:
682 elif opt
in ["backup", "list", "prune"]:
686 logging
.error("Unknown option: " + opt
)
690 man
= BackupManager(conffn
)
692 logging
.debug("Config: " + str(man
.conf
))
694 if epoch
!=None and not epoch
in man
.conf
.epochs
.keys():
695 logging
.error("Unknown epoch '" + epoch
+ "'.")
699 man
.backup(epoch
, mode
)
702 for b
in sorted(man
.listOldBackups(), key
=lambda b
: b
.date
):
703 print(b
.colAlignedString())
708 except (Config
.ReadError
, configparser
.Error
) as e
:
709 logging
.error("Error: " + e
.message
)