e97dbf5182790944bfd2eb1c26987d6e8503ae2e
2 """A simple backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Modes
= ["full", "incr", "diff"]
21 "hour" : datetime
.timedelta(0, 3600),
22 "day" : datetime
.timedelta(1),
23 "week" : datetime
.timedelta(7),
24 "month" : datetime
.timedelta(31),
25 "year" : datetime
.timedelta(365) }
27 def __init__(self
, unit
=None, mult
=1, mode
="full", numkeeps
=None):
31 self
.numkeeps
= numkeeps
35 return "[unit: " + repr(self
.unit
) + \
36 ", mult:" + repr(self
.mult
) + \
37 ", mode: " + repr(self
.mode
) + \
38 ", numkeeps: " + repr(self
.numkeeps
) + \
39 ", excludes: " + repr(self
.excludes
) + "]"
41 def getTimeDelta(self
):
44 return self
.mult
*Epoch
.units
[self
.unit
]
46 def isRipe(self
, oldest
, now
):
54 if delta
>= self
.getTimeDelta():
57 if self
.unit
== "hour":
58 return abs(now
.hour
- oldest
.hour
) >= mult
59 elif self
.unit
== "day":
60 return abs(now
.day
- oldest
.day
) >= mult
61 elif self
.unit
== "week":
62 return abs(now
.isocalendar()[1] - oldest
.isocalendar()[1]) >= mult
63 elif self
.unit
== "month":
64 return abs(now
.month
- oldest
.month
) >= mult
65 elif self
.unit
== "year":
66 return abs(now
.year
- oldest
.year
) >= mult
72 def parseTimedelta( deltastr
):
73 tokens
= [ s
.strip() for s
in deltastr
.split("*") ]
78 elif len(tokens
) == 2:
82 raise ValueError("Invalid format: '{0}'".format(deltastr
))
84 if not unit
in Epoch
.units
:
85 raise ValueError("Unknown unit '{0}'".format(unit
))
88 raise ValueError("Non-positive factor '{0}' given.".format(mult
))
95 """A fileset has a name and a list of directories."""
96 def __init__(self
, name
, dirs
, excludes
):
99 self
.excludes
= excludes
102 return "[name: " + self
.name
+ \
103 ", dirs: " + str(self
.dirs
) + \
104 ", excludes: " + str(self
.excludes
) + "]"
108 """A single backup has a date, an epoch and a mode."""
110 def __init__(self
, date
, epoch
, mode
):
117 def fromDirName(dirname
):
118 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
120 if not mode
in Modes
:
121 raise ValueError("Invalid mode: " + mode
)
123 date
= datetime
.datetime(int(strdate
[0:4]),
124 int(strdate
[4:6]), int(strdate
[6:8]),\
125 int(strtime
[0:2]), int(strtime
[2:4]))
127 return Backup(date
, epoch
, mode
)
130 return "[date: " + self
.date
.ctime() + \
131 ", epoch: " + self
.epoch
+ \
132 ", mode: " + self
.mode
+ "]"
134 def colAlignedString(self
):
135 age
= datetime
.datetime
.now() - self
.date
136 total_hours
= age
.total_seconds()/3600
137 if total_hours
<= 48:
138 agestr
= "(%s h)" % int(total_hours
)
140 agestr
= "(%s d)" % age
.days
141 return "%16s %7s %10s %4s" % (
142 self
.date
.strftime("%Y-%m-%d %H:%M"), agestr
,
143 self
.epoch
, self
.mode
)
146 def getDirName(date
, epoch
, mode
):
147 """Get directory name of backup by given properties."""
148 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
151 def isBackupDir(dirname
):
152 """Is directory a backup directory?"""
153 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
154 return p
.match(dirname
)
159 """Encapsules the configuration for the backup program."""
161 class ReadError(RuntimeError):
162 """An exception raised when reading configurations."""
163 def __init__(self
, value
):
168 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
170 # Filename where checksum of config is saved
171 checksumfn
= "checksum"
174 self
.directory
= None
175 self
.format
= self
.formats
[0]
179 self
.lastchecksum
= None
180 self
.epochs
= Epochs
= { "sporadic" : Epoch() }
184 return "[directory: " + self
.directory
+ \
185 ", format: " + self
.format
+ \
186 ", excludes: " + repr(self
.excludes
) + \
187 ", epochs: " + repr(self
.epochs
) + \
188 ", sets: " + repr(self
.sets
) + "]"
190 def getRealEpochsSorted(self
):
191 """Return all epochs with have a non-None unit, sorted by
192 Epoch.getTimeDelta(), starting with the longest dela."""
194 realepochs
= [ e
for e
in epochs
.keys() if epochs
[e
].unit
!= None ]
195 deltakey
= lambda e
: epochs
[e
].getTimeDelta()
196 realepochs
.sort(key
=deltakey
, reverse
=True)
200 def _read_destination(self
, config
, sec
):
201 for opt
in config
.options(sec
):
203 self
.directory
= config
.get(sec
, opt
)
204 if not os
.path
.isdir(self
.directory
):
205 raise Config
.ReadError("Directory '{0}' does not exist.".format(self
.directory
))
207 self
.format
= config
.get(sec
, opt
)
208 if not self
.format
in Config
.formats
:
209 raise Config
.ReadError("Invalid 'format' given.")
211 raise Config
.ReadError("Unknown option '{0}'.".format(opt
))
214 def _read_global(self
, config
, sec
):
215 for opt
in config
.options(sec
):
216 if opt
.startswith("exclude"):
217 self
.excludes
+= [ config
.get(sec
, opt
) ]
219 raise Config
.ReadError("Unknown option '{0}'.".format(opt
))
222 def _read_epoch(self
, config
, sec
):
223 name
= sec
[6:].strip()
225 if name
in self
.epochs
:
226 raise Config
.ReadError("Epoch '{0}' already defined.".format(name
))
227 if name
in Epoch
.units
:
230 for opt
in config
.options(sec
):
233 e
.numkeeps
= int(config
.getint(sec
, opt
))
235 raise Config
.ReadError("Invalid integer given for '{0}'.".format(opt
))
237 raise Config
.ReadError("Non-positive numkeeps '{0}' given.".format(e
.numkeeps
))
240 e
.mode
= config
.get(sec
, opt
)
241 if not e
.mode
in Modes
:
242 raise Config
.ReadError("Invalid mode '{0}'.".format(e
.mode
))
244 elif opt
=="timedelta":
245 if name
in Epoch
.units
:
246 raise Config
.ReadError("The time delta of a standard epoch " + \
247 "is not supposed to be redefined. ")
248 td
= config
.get(sec
,opt
)
250 mult
, unit
= Epoch
.parseTimedelta(td
)
253 except ValueError as e
:
254 raise Config
.ReadError("Invalid timedelta '{0}': {1}".format(td
, str(e
)))
256 elif opt
.startswith("exclude"):
257 e
.excludes
+= [config
.get(sec
, opt
)]
260 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
262 if e
.numkeeps
== None:
263 raise Config
.ReadError("No numkeeps set for epoch '{0}'.".format(name
))
265 self
.epochs
[name
] = e
268 def _read_set(self
, config
, sec
):
269 name
= sec
[4:].strip()
273 for opt
in config
.options(sec
):
274 if opt
.startswith("dir"):
275 dirs
+= [config
.get(sec
, opt
)]
276 elif opt
.startswith("exclude"):
277 excludes
+= [config
.get(sec
,opt
)]
279 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
281 self
.sets
+= [FileSet(name
, dirs
, excludes
)]
284 def read(self
, filename
):
285 """Read configuration from file"""
287 if not os
.path
.isfile(filename
):
288 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
290 config
= configparser
.RawConfigParser()
291 config
.read(filename
)
293 for reqsec
in ["destination"]:
294 if not config
.has_section(reqsec
):
295 raise Config
.ReadError("Mandatory section '" + reqsec
+ "' is missing.")
297 for sec
in config
.sections():
299 if sec
=="destination":
300 self
._read
_destination
(config
, sec
)
303 self
._read
_global
(config
, sec
)
305 elif sec
.startswith("epoch "):
306 self
._read
_epoch
(config
, sec
)
308 elif sec
.startswith("set "):
309 self
._read
_set
(config
, sec
)
312 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
314 if self
.directory
== None:
315 raise Config
.ReadError("No destination directory set.")
318 # Compute checksum of config file
320 f
= open(filename
, 'rb')
323 self
.checksum
= m
.hexdigest()
328 f
= open(os
.path
.join(self
.directory
, self
.checksumfn
), 'r')
329 self
.lastchecksum
= f
.read().strip()
332 self
.lastchecksum
= None
336 """List and create backups"""
338 def __init__(self
, conffn
):
340 self
.conf
.read(conffn
)
343 def listAllDirs(self
):
344 """List all dirs in destination directory"""
347 basedir
= self
.conf
.directory
348 dirs
= os
.listdir(basedir
)
350 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
353 def listOldBackups(self
):
354 """Returns a list of old backups."""
358 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
359 backups
+= [ Backup
.fromDirName(entry
) ]
364 def getDesiredEpochs(self
, backups
, now
):
365 """Get desired epoch based on self.configuration and list of old backups"""
367 # Find the longest epoch for which we would like the make a backup
368 latest
= datetime
.datetime(1900, 1, 1)
369 for e
in self
.conf
.getRealEpochsSorted():
370 epoch
= self
.conf
.epochs
[e
]
371 if epoch
.numkeeps
<= 0:
374 # Get backups of that epoch
375 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
376 key
=lambda b
: b
.date
))
378 # If there are any, determine the latest
380 latest
= max(latest
, byepoch
[-1].date
)
382 if epoch
.isRipe(latest
, now
):
385 # No backup is to be made
390 def backupFileSet(self
, fileset
, targetdir
, excludes
, since
=None):
391 """Create an archive for given fileset at given target directory."""
393 logfile
= logging
.getLogger('backuplog')
394 logfile
.info("Running file set: " + fileset
.name
)
397 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
401 # Add the since date, if given
403 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
405 # Add the exclude patterns
407 taropts
+= ["--exclude", pat
]
409 #Add exclude patterns from fileset
410 for pat
in fileset
.excludes
:
411 taropts
+= ["--exclude", pat
]
413 # Adding directories to backup
414 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
416 # Launch the tar process
417 tarargs
= [tarpath
] + ["-cpvaf", fsfn
] + taropts
418 logfile
.debug("tar call: " + " ".join(tarargs
))
419 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
420 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
422 # Change tarp's stdout and stderr to non-blocking
423 for s
in [tarp
.stdout
, tarp
.stderr
]:
425 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
426 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
428 # Read stdout and stderr of tarp
430 while tarp
.poll() == None:
431 rd
,wr
,ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
432 if tarp
.stdout
in rd
:
433 logging
.debug( tarp
.stdout
.readline()[:-1].decode() )
434 if tarp
.stderr
in rd
:
435 errmsg
+= tarp
.stderr
.read()
437 # Get the remainging output of tarp
438 for l
in tarp
.stdout
.readlines():
439 logging
.debug(l
.decode().rstrip())
440 errmsg
+= tarp
.stderr
.read()
442 # Get return code of tarp
445 for l
in errmsg
.decode().split("\n"):
447 logfile
.error(tarpath
+ " returned with exit status " + str(rett
) + ".")
450 def backup(self
, epoch
=None, mode
=None):
451 """Make a new backup, if necessary. If epoch is None then determine
452 desired epoch automatically. Use given epoch otherwise. If mode is None
453 then use mode for given epoch. Use given mode otherwise."""
455 now
= datetime
.datetime
.now()
456 oldbackups
= self
.listOldBackups()
458 # Get epoch of backup
460 epoch
= self
.getDesiredEpochs(oldbackups
, now
)
462 logging
.info("No backup planned.")
467 mode
= self
.conf
.epochs
[epoch
].mode
468 logging
.info("Making a backup. Epochs: " + epoch
+ ", mode: " + mode
)
470 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
472 # No old full backups existing
473 if mode
!= "full" and len(oldfullbackups
)==0:
474 logging
.info("No full backups existing. Making a full backup.")
476 # Checksum changed -> self.config file changed
477 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
478 logging
.warning("Full backup recommended as config file has changed.")
481 # If we have a full backup, we backup everything
484 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
486 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
489 logging
.debug("Making backup relative to " + since
.ctime())
491 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
495 # Create new target directory
496 basedir
= self
.conf
.directory
497 dirname
= Backup
.getDirName(now
, epoch
, mode
)
498 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
499 targetdir
= os
.path
.join(basedir
, tmpdirname
)
500 os
.mkdir( targetdir
)
504 logfile
= logging
.getLogger("backuplog")
505 fil
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
506 fil
.setLevel(logging
.DEBUG
)
507 logfile
.addHandler(fil
)
509 logfile
.info("Started: " + now
.ctime())
511 # Backup all file sets
512 for s
in self
.conf
.sets
:
513 excludes
= self
.conf
.excludes
+ self
.conf
.epochs
[epoch
].excludes
514 self
.backupFileSet(s
, targetdir
, excludes
, since
)
516 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
518 # Rename backup directory to final name
519 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
521 # We made a full backup -- recall checksum of config
523 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
524 f
.write( self
.conf
.checksum
)
530 """Prune old backup files"""
532 allDirs
= sorted(self
.listAllDirs())
533 # Collect all directories not matching backup name
534 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
536 # Get all directories which are kept
537 backups
= self
.listOldBackups()
539 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
540 key
=lambda b
: b
.date
, reverse
=True)) for e
in self
.conf
.getRealEpochsSorted() }
542 epoch
= self
.conf
.epochs
[e
]
543 old
= byepoch
[e
][epoch
.numkeeps
:]
544 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
547 logging
.info("List of stale/outdated entries:")
555 if Backup
.isBackupDir(d
):
556 msg
+= Backup
.fromDirName(d
).colAlignedString()
562 # Check that dirs to be removed is in list of all dirs
564 assert( d
in allDirs
)
566 if len(removeDirs
) == 0:
567 logging
.info("No stale/outdated entries to remove.")
570 basedir
= self
.conf
.directory
571 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
575 shutil
.rmtree(os
.path
.join(basedir
, d
))
577 logging
.error("Error when removing '%s': %s" % (d
,e
.strerror
) )
580 def ask_user_yesno(self
, question
):
581 if LogConf
.con
.level
<= logging
.INFO
:
582 return input(question
)
588 """Print --help text"""
590 print("shbackup - a simple backup solution.")
593 print(" " + sys
.argv
[0] + " {options} [cmd]")
594 print(" " + sys
.argv
[0] + " --help")
597 print(" backup make a new backup, if necessary")
598 print(" list list all backups (default)")
599 print(" prune prune outdated/old backups")
602 print(" -h, --help print this usage text")
603 print(" -c, --conf <configfile> use given configuration file")
604 print(" default: /etc/shbackup.conf")
605 print(" -e, --epoch <epoch> force to create backup for given epoch:")
606 print(" year, month, week, day, hour, sporadic")
607 print(" -m, --mode <mode> override mode: full, diff, or incr")
608 print(" -v, --verbose be more verbose and interact with user")
609 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
610 print(" error, warning, info, debug")
611 print(" -V, --version print version info")
616 """Encapsulates logging configuration"""
618 con
= logging
.StreamHandler(sys
.stderr
)
622 """Setup logging system"""
623 conlog
= logging
.getLogger()
624 conlog
.setLevel(logging
.DEBUG
)
626 cls
.con
.setLevel(logging
.WARNING
)
627 conlog
.addHandler(cls
.con
)
629 fillog
= logging
.getLogger("backuplog")
630 fillog
.setLevel(logging
.DEBUG
)
633 if __name__
== "__main__":
637 conffn
= "/etc/shbackup.conf"
643 while i
< len(sys
.argv
)-1:
647 if opt
in ["-h", "--help"]:
651 elif opt
in ["-c", "--conf"]:
655 elif opt
in ["-V", "--version"]:
656 print("shbackup " + __version__
)
659 elif opt
in ["-v", "--verbose"]:
660 LogConf
.con
.setLevel(logging
.INFO
)
662 elif opt
in ["--verbosity"]:
665 numlevel
= getattr(logging
, level
.upper(), None)
666 if not isinstance(numlevel
, int):
667 raise ValueError('Invalid verbosity level: %s' % level
)
668 LogConf
.con
.setLevel(numlevel
)
670 elif opt
in ["-m", "--mode"]:
673 if not mode
in Modes
:
674 logging
.error("Unknown mode '" + mode
+ "'.")
677 elif opt
in ["-e", "--epoch"]:
681 elif opt
in ["backup", "list", "prune"]:
685 logging
.error("Unknown option: " + opt
)
689 man
= BackupManager(conffn
)
691 logging
.debug("Config: " + str(man
.conf
))
693 if epoch
!=None and not epoch
in man
.conf
.epochs
.keys():
694 logging
.error("Unknown epoch '" + epoch
+ "'.")
698 man
.backup(epoch
, mode
)
701 for b
in sorted(man
.listOldBackups(), key
=lambda b
: b
.date
):
702 print(b
.colAlignedString())
707 except (Config
.ReadError
, configparser
.Error
) as e
:
708 logging
.error("Error: " + e
.message
)