2 """A simple backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Modes
= ["full", "incr", "diff"]
21 "hour" : datetime
.timedelta(0, 3600),
22 "day" : datetime
.timedelta(1),
23 "week" : datetime
.timedelta(7),
24 "month" : datetime
.timedelta(31),
25 "year" : datetime
.timedelta(365) }
27 def __init__(self
, unit
=None, mult
=1, mode
="full", numkeeps
=None):
31 self
.numkeeps
= numkeeps
35 return "[unit: " + repr(self
.unit
) + \
36 ", mult:" + repr(self
.mult
) + \
37 ", mode: " + repr(self
.mode
) + \
38 ", numkeeps: " + repr(self
.numkeeps
) + \
39 ", excludes: " + repr(self
.excludes
) + "]"
41 def getTimeDelta(self
):
44 return self
.mult
*Epoch
.units
[self
.unit
]
46 def isRipe(self
, oldest
, now
):
54 if delta
>= self
.getTimeDelta():
57 if self
.unit
== "hour":
58 return abs(now
.hour
- oldest
.hour
) >= mult
59 elif self
.unit
== "day":
60 return abs(now
.day
- oldest
.day
) >= mult
61 elif self
.unit
== "week":
62 return abs(now
.isocalendar()[1] - oldest
.isocalendar()[1]) >= mult
63 elif self
.unit
== "month":
64 return abs(now
.month
- oldest
.month
) >= mult
65 elif self
.unit
== "year":
66 return abs(now
.year
- oldest
.year
) >= mult
72 def parseTimedelta( deltastr
):
73 tokens
= [ s
.strip() for s
in deltastr
.split("*") ]
78 elif len(tokens
) == 2:
82 raise ValueError("Invalid format: '{0}'".format(deltastr
))
84 if not unit
in Epoch
.units
:
85 raise ValueError("Unknown unit '{0}'".format(unit
))
88 raise ValueError("Non-positive factor '{0}' given.".format(mult
))
95 """A fileset has a name and a list of directories."""
96 def __init__(self
, name
, dirs
, excludes
):
99 self
.excludes
= excludes
102 return "[name: " + self
.name
+ \
103 ", dirs: " + str(self
.dirs
) + \
104 ", excludes: " + str(self
.excludes
) + "]"
108 """A single backup has a date, an epoch and a mode."""
110 def __init__(self
, date
, epoch
, mode
):
117 def fromDirName(dirname
):
118 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
120 if not mode
in Modes
:
121 raise ValueError("Invalid mode: " + mode
)
123 date
= datetime
.datetime(int(strdate
[0:4]),
124 int(strdate
[4:6]), int(strdate
[6:8]),\
125 int(strtime
[0:2]), int(strtime
[2:4]))
127 return Backup(date
, epoch
, mode
)
130 return "[date: " + self
.date
.ctime() + \
131 ", epoch: " + self
.epoch
+ \
132 ", mode: " + self
.mode
+ "]"
134 def colAlignedString(self
):
135 age
= datetime
.datetime
.now() - self
.date
136 total_hours
= age
.total_seconds()/3600
137 if total_hours
<= 48:
138 agestr
= "(%s h)" % int(total_hours
)
140 agestr
= "(%s d)" % age
.days
141 return "%16s %7s %10s %4s" % (
142 self
.date
.strftime("%Y-%m-%d %H:%M"), agestr
,
143 self
.epoch
, self
.mode
)
146 def getDirName(date
, epoch
, mode
):
147 """Get directory name of backup by given properties."""
148 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
151 def isBackupDir(dirname
):
152 """Is directory a backup directory?"""
153 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
154 return p
.match(dirname
)
159 """Encapsules the configuration for the backup program."""
161 class ReadError(RuntimeError):
162 """An exception raised when reading configurations."""
163 def __init__(self
, value
):
168 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
170 # Filename where checksum of config is saved
171 checksumfn
= "checksum"
174 self
.backupdir
= None
175 self
.format
= self
.formats
[1]
176 self
.tarbin
= "/bin/tar"
180 self
.lastchecksum
= None
181 self
.epochs
= Epochs
= { "sporadic" : Epoch() }
185 return "[backupdir: " + self
.backupdir
+ \
186 ", format: " + self
.format
+ \
187 ", tarbin: " + self
.tarbin
+ \
188 ", excludes: " + repr(self
.excludes
) + \
189 ", epochs: " + repr(self
.epochs
) + \
190 ", sets: " + repr(self
.sets
) + "]"
192 def getRealEpochsSorted(self
):
193 """Return all epochs with have a non-None unit, sorted by
194 Epoch.getTimeDelta(), starting with the longest dela."""
196 realepochs
= [ e
for e
in epochs
.keys() if epochs
[e
].unit
!= None ]
197 deltakey
= lambda e
: epochs
[e
].getTimeDelta()
198 realepochs
.sort(key
=deltakey
, reverse
=True)
202 def _read_global(self
, config
, sec
):
203 for opt
in config
.options(sec
):
205 self
.backupdir
= config
.get(sec
, opt
)
206 if not os
.path
.isdir(self
.backupdir
):
207 raise Config
.ReadError("Backupdir '{0}' does not exist.".format(self
.backupdir
))
209 self
.format
= config
.get(sec
, opt
)
210 if not self
.format
in Config
.formats
:
211 raise Config
.ReadError("Invalid 'format' given.")
213 self
.tarbin
= config
.get(sec
, opt
)
214 if not os
.path
.isfile(self
.tarbin
):
215 raise Config
.ReadError("Tar binary '{0}' does not exist.".format(self
.tarbin
))
216 elif opt
.startswith("exclude"):
217 self
.excludes
+= [ config
.get(sec
, opt
) ]
219 raise Config
.ReadError("Unknown option '{0}'.".format(opt
))
222 def _read_epoch(self
, config
, sec
):
223 name
= sec
[6:].strip()
225 if name
in self
.epochs
:
226 raise Config
.ReadError("Epoch '{0}' already defined.".format(name
))
227 p
= re
.compile(r
'^\w+$')
228 if not p
.match(name
):
229 raise Config
.ReadError("Epoch name '{0}' does not only " + \
230 "comprise alphanumeric characters.".format(name
))
231 if name
in Epoch
.units
:
234 for opt
in config
.options(sec
):
237 e
.numkeeps
= int(config
.getint(sec
, opt
))
239 raise Config
.ReadError("Invalid integer given for '{0}'.".format(opt
))
241 raise Config
.ReadError("Non-positive numkeeps '{0}' given.".format(e
.numkeeps
))
244 e
.mode
= config
.get(sec
, opt
)
245 if not e
.mode
in Modes
:
246 raise Config
.ReadError("Invalid mode '{0}'.".format(e
.mode
))
248 elif opt
=="timespan":
249 if name
in Epoch
.units
:
250 raise Config
.ReadError("The time delta of a standard epoch " + \
251 "is not supposed to be redefined. ")
252 td
= config
.get(sec
,opt
)
254 mult
, unit
= Epoch
.parseTimedelta(td
)
257 except ValueError as e
:
258 raise Config
.ReadError("Invalid timespan '{0}': {1}".format(td
, str(e
)))
260 elif opt
.startswith("exclude"):
261 e
.excludes
+= [config
.get(sec
, opt
)]
264 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
266 if e
.numkeeps
== None:
267 raise Config
.ReadError("No numkeeps set for epoch '{0}'.".format(name
))
269 self
.epochs
[name
] = e
272 def _read_set(self
, config
, sec
):
273 name
= sec
[4:].strip()
274 p
= re
.compile(r
'^\w+$')
275 if not p
.match(name
):
276 raise Config
.ReadError("Set name '{0}' does not only " + \
277 "comprise alphanumeric characters.".format(name
))
282 for opt
in config
.options(sec
):
283 if opt
.startswith("dir"):
284 dirs
+= [config
.get(sec
, opt
)]
285 elif opt
.startswith("exclude"):
286 excludes
+= [config
.get(sec
,opt
)]
288 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
290 self
.sets
+= [FileSet(name
, dirs
, excludes
)]
293 def read(self
, filename
):
294 """Read configuration from file"""
296 if not os
.path
.isfile(filename
):
297 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
299 config
= configparser
.RawConfigParser()
300 config
.read(filename
)
302 for reqsec
in ["global"]:
303 if not config
.has_section(reqsec
):
304 raise Config
.ReadError("Mandatory section '" + reqsec
+ "' is missing.")
306 for sec
in config
.sections():
309 self
._read
_global
(config
, sec
)
311 elif sec
.startswith("epoch "):
312 self
._read
_epoch
(config
, sec
)
314 elif sec
.startswith("set "):
315 self
._read
_set
(config
, sec
)
318 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
320 if self
.backupdir
== None:
321 raise Config
.ReadError("No backup directory set.")
324 # Compute checksum of config file
326 f
= open(filename
, 'rb')
329 self
.checksum
= m
.hexdigest()
334 f
= open(os
.path
.join(self
.backupdir
, self
.checksumfn
), 'r')
335 self
.lastchecksum
= f
.read().strip()
338 self
.lastchecksum
= None
342 """List and create backups"""
344 def __init__(self
, conffn
):
346 self
.conf
.read(conffn
)
349 def listAllDirs(self
):
350 """List all dirs in backupdir"""
353 basedir
= self
.conf
.backupdir
354 dirs
= os
.listdir(basedir
)
356 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
359 def listExistingBackups(self
):
360 """Returns a list of old backups."""
364 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
365 backups
+= [ Backup
.fromDirName(entry
) ]
370 def getDesiredEpochs(self
, backups
, now
):
371 """Get desired epoch based on self.configuration and list of old backups"""
373 # Find the longest epoch for which we would like the make a backup
374 latest
= datetime
.datetime(1900, 1, 1)
375 for e
in self
.conf
.getRealEpochsSorted():
376 epoch
= self
.conf
.epochs
[e
]
377 if epoch
.numkeeps
<= 0:
380 # Get backups of that epoch
381 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
382 key
=lambda b
: b
.date
))
384 # If there are any, determine the latest
386 latest
= max(latest
, byepoch
[-1].date
)
388 if epoch
.isRipe(latest
, now
):
391 # No backup is to be made
396 def backupFileSet(self
, fileset
, targetdir
, excludes
, since
=None):
397 """Create an archive for given fileset at given target directory."""
399 logfile
= logging
.getLogger('backuplog')
400 logfile
.info("Running file set: " + fileset
.name
)
402 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
405 # Tar is verbose is sitarba is verbose
406 if LogConf
.con
.level
<= logging
.DEBUG
:
407 taropts
+= ["--verbose"]
409 # Add the since date, if given
411 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
413 # Add the exclude patterns
415 taropts
+= ["--exclude", pat
]
417 #Add exclude patterns from fileset
418 for pat
in fileset
.excludes
:
419 taropts
+= ["--exclude", pat
]
422 # Adding directories to backup
423 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
425 # Launch the tar process
426 tarargs
= [self
.conf
.tarbin
] + ["-cpaf", fsfn
] + taropts
427 logfile
.debug("tar call: " + " ".join(tarargs
))
428 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
429 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
431 # Change tarp's stdout and stderr to non-blocking
432 for s
in [tarp
.stdout
, tarp
.stderr
]:
434 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
435 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
437 # Read stdout and stderr of tarp
439 while tarp
.poll() == None:
440 rd
,wr
,ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
441 if tarp
.stdout
in rd
:
442 logging
.debug( tarp
.stdout
.readline()[:-1].decode() )
443 if tarp
.stderr
in rd
:
444 errmsg
+= tarp
.stderr
.read()
446 # Get the remainging output of tarp
447 for l
in tarp
.stdout
.readlines():
448 logging
.debug(l
.decode().rstrip())
449 errmsg
+= tarp
.stderr
.read()
451 # Get return code of tarp
454 for l
in errmsg
.decode().split("\n"):
456 logfile
.error(self
.conf
.tarbin
+ " returned with exit status " + \
460 def backup(self
, epoch
=None, mode
=None):
461 """Make a new backup, if necessary. If epoch is None then determine
462 desired epoch automatically. Use given epoch otherwise. If mode is None
463 then use mode for given epoch. Use given mode otherwise."""
465 now
= datetime
.datetime
.now()
466 oldbackups
= self
.listExistingBackups()
468 # Get epoch of backup
470 epoch
= self
.getDesiredEpochs(oldbackups
, now
)
472 logging
.info("No backup planned.")
477 mode
= self
.conf
.epochs
[epoch
].mode
478 logging
.info("Making a backup. Epochs: " + epoch
+ ", mode: " + mode
)
480 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
482 # No old full backups existing
483 if mode
!= "full" and len(oldfullbackups
)==0:
484 logging
.info("No full backups existing. Making a full backup.")
486 # Checksum changed -> self.config file changed
487 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
488 logging
.warning("Full backup recommended as config file has changed.")
491 # If we have a full backup, we backup everything
494 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
496 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
499 logging
.debug("Making backup relative to " + since
.ctime())
501 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
505 # Create new backup directory
506 basedir
= self
.conf
.backupdir
507 dirname
= Backup
.getDirName(now
, epoch
, mode
)
508 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
509 targetdir
= os
.path
.join(basedir
, tmpdirname
)
510 os
.mkdir( targetdir
)
514 logfile
= logging
.getLogger("backuplog")
515 fil
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
516 fil
.setLevel(logging
.DEBUG
)
517 logfile
.addHandler(fil
)
519 logfile
.info("Started: " + now
.ctime())
521 # Backup all file sets
522 for s
in self
.conf
.sets
:
523 excludes
= self
.conf
.excludes
+ self
.conf
.epochs
[epoch
].excludes
524 self
.backupFileSet(s
, targetdir
, excludes
, since
)
526 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
528 # Rename backup directory to final name
529 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
531 # We made a full backup -- recall checksum of config
533 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
534 f
.write( self
.conf
.checksum
)
540 """Prune old backup files"""
542 allDirs
= sorted(self
.listAllDirs())
543 # Collect all directories that are removed
544 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
547 backups
= self
.listExistingBackups()
548 # Group backups by epoch and sort them by age
549 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
550 key
=lambda b
: b
.date
, reverse
=True)) \
551 for e
in self
.conf
.getRealEpochsSorted() }
552 # If we have too many backups of a specific epoch --> add them to remove list
554 epoch
= self
.conf
.epochs
[e
]
555 old
= byepoch
[e
][epoch
.numkeeps
:]
556 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
559 logging
.info("List of stale/outdated entries:")
567 if Backup
.isBackupDir(d
):
568 msg
+= Backup
.fromDirName(d
).colAlignedString()
574 # Check that dirs to be removed is in list of all dirs
576 assert( d
in allDirs
)
578 if len(removeDirs
) == 0:
579 logging
.info("No stale/outdated entries to remove.")
582 basedir
= self
.conf
.backupdir
583 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
587 shutil
.rmtree(os
.path
.join(basedir
, d
))
589 logging
.error("Error when removing '%s': %s" % (d
,e
.strerror
) )
592 def ask_user_yesno(self
, question
):
593 if LogConf
.con
.level
<= logging
.INFO
:
594 return input(question
)
600 """Print --help text"""
602 print("sitarba - a simple backup solution.")
605 print(" " + sys
.argv
[0] + " {options} [cmd]")
606 print(" " + sys
.argv
[0] + " --help")
609 print(" backup make a new backup, if necessary")
610 print(" list list all backups (default)")
611 print(" prune prune outdated/old backups")
614 print(" -h, --help print this usage text")
615 print(" -c, --conf FILE use given configuration file")
616 print(" default: /etc/sitarba.conf")
617 print(" -e, --epoch EPOCH force to create backup for given epoch, which")
618 print(" can be 'sporadic' or one of the configured epochs")
619 print(" -m, --mode MODE override mode: full, diff, or incr")
620 print(" -v, --verbose be more verbose and interact with user")
621 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
622 print(" error, warning, info, debug")
623 print(" -V, --version print version info")
628 """Encapsulates logging configuration"""
630 con
= logging
.StreamHandler(sys
.stderr
)
634 """Setup logging system"""
635 conlog
= logging
.getLogger()
636 conlog
.setLevel(logging
.DEBUG
)
638 cls
.con
.setLevel(logging
.WARNING
)
639 conlog
.addHandler(cls
.con
)
641 fillog
= logging
.getLogger("backuplog")
642 fillog
.setLevel(logging
.DEBUG
)
645 if __name__
== "__main__":
649 conffn
= "/etc/sitarba.conf"
655 while i
< len(sys
.argv
)-1:
659 if opt
in ["-h", "--help"]:
663 elif opt
in ["-c", "--conf"]:
667 elif opt
in ["-V", "--version"]:
668 print("sitarba " + __version__
)
671 elif opt
in ["-v", "--verbose"]:
672 LogConf
.con
.setLevel(logging
.INFO
)
674 elif opt
in ["--verbosity"]:
677 numlevel
= getattr(logging
, level
.upper(), None)
678 if not isinstance(numlevel
, int):
679 raise ValueError('Invalid verbosity level: %s' % level
)
680 LogConf
.con
.setLevel(numlevel
)
682 elif opt
in ["-m", "--mode"]:
685 if not mode
in Modes
:
686 logging
.error("Unknown mode '" + mode
+ "'.")
689 elif opt
in ["-e", "--epoch"]:
693 elif opt
in ["backup", "list", "prune"]:
697 logging
.error("Unknown option: " + opt
)
701 man
= BackupManager(conffn
)
703 logging
.debug("Config: " + str(man
.conf
))
705 if epoch
!=None and not epoch
in man
.conf
.epochs
.keys():
706 logging
.error("Unknown epoch '" + epoch
+ "'.")
710 man
.backup(epoch
, mode
)
713 for b
in sorted(man
.listExistingBackups(), key
=lambda b
: b
.date
):
714 print(b
.colAlignedString())
719 except (Config
.ReadError
, configparser
.Error
) as e
:
720 logging
.error("Error: " + e
.message
)