2 """A simple backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Modes
= ["full", "incr", "diff"]
21 "hour" : datetime
.timedelta(0, 3600),
22 "day" : datetime
.timedelta(1),
23 "week" : datetime
.timedelta(7),
24 "month" : datetime
.timedelta(31),
25 "year" : datetime
.timedelta(365) }
27 def __init__(self
, unit
=None, mult
=1, mode
="full", numkeeps
=None):
31 self
.numkeeps
= numkeeps
35 return "[unit: " + repr(self
.unit
) + \
36 ", mult:" + repr(self
.mult
) + \
37 ", mode: " + repr(self
.mode
) + \
38 ", numkeeps: " + repr(self
.numkeeps
) + \
39 ", excludes: " + repr(self
.excludes
) + "]"
41 def getTimeDelta(self
):
44 return self
.mult
*Epoch
.units
[self
.unit
]
46 def isRipe(self
, oldest
, now
):
54 if delta
>= self
.getTimeDelta():
57 if self
.unit
== "hour":
58 return abs(now
.hour
- oldest
.hour
) >= mult
59 elif self
.unit
== "day":
60 return abs(now
.day
- oldest
.day
) >= mult
61 elif self
.unit
== "week":
62 return abs(now
.isocalendar()[1] - oldest
.isocalendar()[1]) >= mult
63 elif self
.unit
== "month":
64 return abs(now
.month
- oldest
.month
) >= mult
65 elif self
.unit
== "year":
66 return abs(now
.year
- oldest
.year
) >= mult
72 def parseTimedelta( deltastr
):
73 tokens
= [ s
.strip() for s
in deltastr
.split("*") ]
78 elif len(tokens
) == 2:
82 raise ValueError("Invalid format: '{0}'".format(deltastr
))
84 if not unit
in Epoch
.units
:
85 raise ValueError("Unknown unit '{0}'".format(unit
))
88 raise ValueError("Non-positive factor '{0}' given.".format(mult
))
95 """A fileset has a name and a list of directories."""
96 def __init__(self
, name
, dirs
, excludes
):
99 self
.excludes
= excludes
102 return "[name: " + self
.name
+ \
103 ", dirs: " + str(self
.dirs
) + \
104 ", excludes: " + str(self
.excludes
) + "]"
108 """A single backup has a date, an epoch and a mode."""
110 def __init__(self
, date
, epoch
, mode
):
117 def fromDirName(dirname
):
118 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
120 if not mode
in Modes
:
121 raise ValueError("Invalid mode: " + mode
)
123 date
= datetime
.datetime(int(strdate
[0:4]),
124 int(strdate
[4:6]), int(strdate
[6:8]),\
125 int(strtime
[0:2]), int(strtime
[2:4]))
127 return Backup(date
, epoch
, mode
)
130 return "[date: " + self
.date
.ctime() + \
131 ", epoch: " + self
.epoch
+ \
132 ", mode: " + self
.mode
+ "]"
134 def colAlignedString(self
):
135 age
= datetime
.datetime
.now() - self
.date
136 total_hours
= age
.total_seconds()/3600
137 if total_hours
<= 48:
138 agestr
= "(%s h)" % int(total_hours
)
140 agestr
= "(%s d)" % age
.days
141 return "%16s %7s %10s %4s" % (
142 self
.date
.strftime("%Y-%m-%d %H:%M"), agestr
,
143 self
.epoch
, self
.mode
)
146 def getDirName(date
, epoch
, mode
):
147 """Get directory name of backup by given properties."""
148 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
151 def isBackupDir(dirname
):
152 """Is directory a backup directory?"""
153 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
154 return p
.match(dirname
)
159 """Encapsules the configuration for the backup program."""
161 class ReadError(RuntimeError):
162 """An exception raised when reading configurations."""
163 def __init__(self
, value
):
168 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
170 # Filename where checksum of config is saved
171 checksumfn
= "checksum"
174 self
.backupdir
= None
175 self
.format
= self
.formats
[1]
176 self
.tarbin
= "/bin/tar"
180 self
.lastchecksum
= None
181 self
.epochs
= Epochs
= { "sporadic" : Epoch() }
185 return "[backupdir: " + self
.backupdir
+ \
186 ", format: " + self
.format
+ \
187 ", tarbin: " + self
.tarbin
+ \
188 ", excludes: " + repr(self
.excludes
) + \
189 ", epochs: " + repr(self
.epochs
) + \
190 ", sets: " + repr(self
.sets
) + "]"
192 def getRealEpochsSorted(self
):
193 """Return all epochs with have a non-None unit, sorted by
194 Epoch.getTimeDelta(), starting with the longest dela."""
196 realepochs
= [ e
for e
in epochs
.keys() if epochs
[e
].unit
!= None ]
197 deltakey
= lambda e
: epochs
[e
].getTimeDelta()
198 realepochs
.sort(key
=deltakey
, reverse
=True)
202 def _read_global(self
, config
, sec
):
203 for opt
in config
.options(sec
):
205 self
.backupdir
= config
.get(sec
, opt
)
206 if not os
.path
.isdir(self
.backupdir
):
207 raise Config
.ReadError("Backupdir '{0}' does not exist.".format(self
.backupdir
))
209 self
.format
= config
.get(sec
, opt
)
210 if not self
.format
in Config
.formats
:
211 raise Config
.ReadError("Invalid 'format' given.")
213 self
.tarbin
= config
.get(sec
, opt
)
214 if not os
.path
.isfile(self
.tarbin
):
215 raise Config
.ReadError("Tar binary '{0}' does not exist.".format(self
.tarbin
))
216 elif opt
.startswith("exclude"):
217 self
.excludes
+= [ config
.get(sec
, opt
) ]
219 raise Config
.ReadError("Unknown option '{0}'.".format(opt
))
222 def _read_epoch(self
, config
, sec
):
223 name
= sec
[6:].strip()
225 if name
in self
.epochs
:
226 raise Config
.ReadError("Epoch '{0}' already defined.".format(name
))
227 p
= re
.compile(r
'^\w+$')
228 if not p
.match(name
):
229 raise Config
.ReadError("Epoch name '{0}' does not only " + \
230 "comprise alphanumeric characters.".format(name
))
231 if name
in Epoch
.units
:
234 for opt
in config
.options(sec
):
237 e
.numkeeps
= int(config
.getint(sec
, opt
))
239 raise Config
.ReadError("Invalid integer given for '{0}'.".format(opt
))
241 raise Config
.ReadError("Non-positive numkeeps '{0}' given.".format(e
.numkeeps
))
244 e
.mode
= config
.get(sec
, opt
)
245 if not e
.mode
in Modes
:
246 raise Config
.ReadError("Invalid mode '{0}'.".format(e
.mode
))
248 elif opt
=="timespan":
249 if name
in Epoch
.units
:
250 raise Config
.ReadError("The time delta of a standard epoch " + \
251 "is not supposed to be redefined. ")
252 td
= config
.get(sec
,opt
)
254 mult
, unit
= Epoch
.parseTimedelta(td
)
257 except ValueError as e
:
258 raise Config
.ReadError("Invalid timespan '{0}': {1}".format(td
, str(e
)))
260 elif opt
.startswith("exclude"):
261 e
.excludes
+= [config
.get(sec
, opt
)]
264 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
266 if e
.numkeeps
== None:
267 raise Config
.ReadError("No numkeeps set for epoch '{0}'.".format(name
))
269 self
.epochs
[name
] = e
272 def _read_set(self
, config
, sec
):
273 name
= sec
[4:].strip()
274 p
= re
.compile(r
'^\w+$')
275 if not p
.match(name
):
276 raise Config
.ReadError("Set name '{0}' does not only " + \
277 "comprise alphanumeric characters.".format(name
))
282 for opt
in config
.options(sec
):
283 if opt
.startswith("dir"):
284 dirs
+= [config
.get(sec
, opt
)]
285 elif opt
.startswith("exclude"):
286 excludes
+= [config
.get(sec
,opt
)]
288 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
290 self
.sets
+= [FileSet(name
, dirs
, excludes
)]
293 def read(self
, filename
):
294 """Read configuration from file"""
296 if not os
.path
.isfile(filename
):
297 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
299 config
= configparser
.RawConfigParser()
300 config
.read(filename
)
302 for reqsec
in ["global"]:
303 if not config
.has_section(reqsec
):
304 raise Config
.ReadError("Mandatory section '" + reqsec
+ "' is missing.")
306 for sec
in config
.sections():
309 self
._read
_global
(config
, sec
)
311 elif sec
.startswith("epoch "):
312 self
._read
_epoch
(config
, sec
)
314 elif sec
.startswith("set "):
315 self
._read
_set
(config
, sec
)
318 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
320 if self
.backupdir
== None:
321 raise Config
.ReadError("No backup directory set.")
324 # Compute checksum of config file
326 f
= open(filename
, 'rb')
329 self
.checksum
= m
.hexdigest()
334 f
= open(os
.path
.join(self
.backupdir
, self
.checksumfn
), 'r')
335 self
.lastchecksum
= f
.read().strip()
338 self
.lastchecksum
= None
342 """List and create backups"""
344 def __init__(self
, conffn
):
346 self
.conf
.read(conffn
)
349 def listAllDirs(self
):
350 """List all dirs in backupdir"""
353 basedir
= self
.conf
.backupdir
354 dirs
= os
.listdir(basedir
)
356 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
359 def listOldBackups(self
):
360 """Returns a list of old backups."""
364 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
365 backups
+= [ Backup
.fromDirName(entry
) ]
370 def getDesiredEpochs(self
, backups
, now
):
371 """Get desired epoch based on self.configuration and list of old backups"""
373 # Find the longest epoch for which we would like the make a backup
374 latest
= datetime
.datetime(1900, 1, 1)
375 for e
in self
.conf
.getRealEpochsSorted():
376 epoch
= self
.conf
.epochs
[e
]
377 if epoch
.numkeeps
<= 0:
380 # Get backups of that epoch
381 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
382 key
=lambda b
: b
.date
))
384 # If there are any, determine the latest
386 latest
= max(latest
, byepoch
[-1].date
)
388 if epoch
.isRipe(latest
, now
):
391 # No backup is to be made
396 def backupFileSet(self
, fileset
, targetdir
, excludes
, since
=None):
397 """Create an archive for given fileset at given target directory."""
399 logfile
= logging
.getLogger('backuplog')
400 logfile
.info("Running file set: " + fileset
.name
)
402 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
405 # Tar is verbose is sitarba is verbose
406 if LogConf
.con
.level
<= logging
.INFO
:
407 taropts
+= ["--verbose"]
409 # Add the since date, if given
411 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
413 # Add the exclude patterns
415 taropts
+= ["--exclude", pat
]
417 #Add exclude patterns from fileset
418 for pat
in fileset
.excludes
:
419 taropts
+= ["--exclude", pat
]
422 # Adding directories to backup
423 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
425 # Launch the tar process
426 tarargs
= [self
.conf
.tarbin
] + ["-cpaf", fsfn
] + taropts
427 logfile
.debug("tar call: " + " ".join(tarargs
))
428 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
429 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
431 # Change tarp's stdout and stderr to non-blocking
432 for s
in [tarp
.stdout
, tarp
.stderr
]:
434 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
435 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
437 # Read stdout and stderr of tarp
439 while tarp
.poll() == None:
440 rd
,wr
,ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
441 if tarp
.stdout
in rd
:
442 logging
.debug( tarp
.stdout
.readline()[:-1].decode() )
443 if tarp
.stderr
in rd
:
444 errmsg
+= tarp
.stderr
.read()
446 # Get the remainging output of tarp
447 for l
in tarp
.stdout
.readlines():
448 logging
.debug(l
.decode().rstrip())
449 errmsg
+= tarp
.stderr
.read()
451 # Get return code of tarp
454 for l
in errmsg
.decode().split("\n"):
456 logfile
.error(self
.conf
.tarbin
+ " returned with exit status " + \
460 def backup(self
, epoch
=None, mode
=None):
461 """Make a new backup, if necessary. If epoch is None then determine
462 desired epoch automatically. Use given epoch otherwise. If mode is None
463 then use mode for given epoch. Use given mode otherwise."""
465 now
= datetime
.datetime
.now()
466 oldbackups
= self
.listOldBackups()
468 # Get epoch of backup
470 epoch
= self
.getDesiredEpochs(oldbackups
, now
)
472 logging
.info("No backup planned.")
477 mode
= self
.conf
.epochs
[epoch
].mode
478 logging
.info("Making a backup. Epochs: " + epoch
+ ", mode: " + mode
)
480 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
482 # No old full backups existing
483 if mode
!= "full" and len(oldfullbackups
)==0:
484 logging
.info("No full backups existing. Making a full backup.")
486 # Checksum changed -> self.config file changed
487 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
488 logging
.warning("Full backup recommended as config file has changed.")
491 # If we have a full backup, we backup everything
494 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
496 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
499 logging
.debug("Making backup relative to " + since
.ctime())
501 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
505 # Create new backup directory
506 basedir
= self
.conf
.backupdir
507 dirname
= Backup
.getDirName(now
, epoch
, mode
)
508 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
509 targetdir
= os
.path
.join(basedir
, tmpdirname
)
510 os
.mkdir( targetdir
)
514 logfile
= logging
.getLogger("backuplog")
515 fil
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
516 fil
.setLevel(logging
.DEBUG
)
517 logfile
.addHandler(fil
)
519 logfile
.info("Started: " + now
.ctime())
521 # Backup all file sets
522 for s
in self
.conf
.sets
:
523 excludes
= self
.conf
.excludes
+ self
.conf
.epochs
[epoch
].excludes
524 self
.backupFileSet(s
, targetdir
, excludes
, since
)
526 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
528 # Rename backup directory to final name
529 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
531 # We made a full backup -- recall checksum of config
533 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
534 f
.write( self
.conf
.checksum
)
540 """Prune old backup files"""
542 allDirs
= sorted(self
.listAllDirs())
543 # Collect all directories not matching backup name
544 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
546 # Get all directories which are kept
547 backups
= self
.listOldBackups()
549 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
550 key
=lambda b
: b
.date
, reverse
=True)) for e
in self
.conf
.getRealEpochsSorted() }
552 epoch
= self
.conf
.epochs
[e
]
553 old
= byepoch
[e
][epoch
.numkeeps
:]
554 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
557 logging
.info("List of stale/outdated entries:")
565 if Backup
.isBackupDir(d
):
566 msg
+= Backup
.fromDirName(d
).colAlignedString()
572 # Check that dirs to be removed is in list of all dirs
574 assert( d
in allDirs
)
576 if len(removeDirs
) == 0:
577 logging
.info("No stale/outdated entries to remove.")
580 basedir
= self
.conf
.backupdir
581 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
585 shutil
.rmtree(os
.path
.join(basedir
, d
))
587 logging
.error("Error when removing '%s': %s" % (d
,e
.strerror
) )
590 def ask_user_yesno(self
, question
):
591 if LogConf
.con
.level
<= logging
.INFO
:
592 return input(question
)
598 """Print --help text"""
600 print("sitarba - a simple backup solution.")
603 print(" " + sys
.argv
[0] + " {options} [cmd]")
604 print(" " + sys
.argv
[0] + " --help")
607 print(" backup make a new backup, if necessary")
608 print(" list list all backups (default)")
609 print(" prune prune outdated/old backups")
612 print(" -h, --help print this usage text")
613 print(" -c, --conf FILE use given configuration file")
614 print(" default: /etc/sitarba.conf")
615 print(" -e, --epoch EPOCH force to create backup for given epoch, which")
616 print(" can be 'sporadic' or one of the configured epochs")
617 print(" -m, --mode MODE override mode: full, diff, or incr")
618 print(" -v, --verbose be more verbose and interact with user")
619 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
620 print(" error, warning, info, debug")
621 print(" -V, --version print version info")
626 """Encapsulates logging configuration"""
628 con
= logging
.StreamHandler(sys
.stderr
)
632 """Setup logging system"""
633 conlog
= logging
.getLogger()
634 conlog
.setLevel(logging
.DEBUG
)
636 cls
.con
.setLevel(logging
.WARNING
)
637 conlog
.addHandler(cls
.con
)
639 fillog
= logging
.getLogger("backuplog")
640 fillog
.setLevel(logging
.DEBUG
)
643 if __name__
== "__main__":
647 conffn
= "/etc/sitarba.conf"
653 while i
< len(sys
.argv
)-1:
657 if opt
in ["-h", "--help"]:
661 elif opt
in ["-c", "--conf"]:
665 elif opt
in ["-V", "--version"]:
666 print("sitarba " + __version__
)
669 elif opt
in ["-v", "--verbose"]:
670 LogConf
.con
.setLevel(logging
.INFO
)
672 elif opt
in ["--verbosity"]:
675 numlevel
= getattr(logging
, level
.upper(), None)
676 if not isinstance(numlevel
, int):
677 raise ValueError('Invalid verbosity level: %s' % level
)
678 LogConf
.con
.setLevel(numlevel
)
680 elif opt
in ["-m", "--mode"]:
683 if not mode
in Modes
:
684 logging
.error("Unknown mode '" + mode
+ "'.")
687 elif opt
in ["-e", "--epoch"]:
691 elif opt
in ["backup", "list", "prune"]:
695 logging
.error("Unknown option: " + opt
)
699 man
= BackupManager(conffn
)
701 logging
.debug("Config: " + str(man
.conf
))
703 if epoch
!=None and not epoch
in man
.conf
.epochs
.keys():
704 logging
.error("Unknown epoch '" + epoch
+ "'.")
708 man
.backup(epoch
, mode
)
711 for b
in sorted(man
.listOldBackups(), key
=lambda b
: b
.date
):
712 print(b
.colAlignedString())
717 except (Config
.ReadError
, configparser
.Error
) as e
:
718 logging
.error("Error: " + e
.message
)