a45305e59ae8e9f1f88f28126065bfd457618c8c
2 """Stefan Huber's simplistic backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Mode
= ["full", "incr", "diff"]
19 "hour" : datetime
.timedelta(0, 3600), \
20 "day" : datetime
.timedelta(1), \
21 "week" : datetime
.timedelta(7), \
22 "month" : datetime
.timedelta(30), \
23 "year" : datetime
.timedelta(365) }
25 Epoch
= dict(RealEpoch
, **{ \
26 "sporadic" : datetime
.timedelta(0,0) \
31 """A single backup has a date, an epoch and a mode."""
33 def __init__(self
, date
, epoch
, mode
):
39 def fromDirName(dirname
):
40 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
42 if not epoch
in Epoch
.keys():
43 raise ValueError("Invalid epoch: " + epoch
)
46 raise ValueError("Invalid mode: " + mode
)
48 date
= datetime
.datetime(int(strdate
[0:4]),
49 int(strdate
[4:6]), int(strdate
[6:8]),\
50 int(strtime
[0:2]), int(strtime
[2:4]))
52 return Backup(date
, epoch
, mode
)
55 return "[date: " + self
.date
.ctime() + \
56 ", epoch: " + self
.epoch
+ \
57 ", mode: " + self
.mode
+ "]"
59 def colAlignedString(self
):
60 return "%16s %8s %4s" % ( \
61 self
.date
.strftime("%Y-%m-%d %H:%M"), self
.epoch
, self
.mode
)
64 def getDirName(date
, epoch
, mode
):
65 """Get directory name of backup by given properties."""
66 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
69 def isBackupDir(dirname
):
70 """Is directory a backup directory?"""
71 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
72 return p
.match(dirname
)
77 """Encapsules the configuration for the backup program."""
79 class ReadError(RuntimeError):
80 """An exception raised when reading configurations."""
81 def __init__(self
, value
):
86 """A fileset has a name and a list of directories."""
87 def __init__(self
, name
, dirs
):
92 return "[name: " + self
.name
+ ", dirs: " + str(self
.dirs
) + "]"
94 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
96 # Filename where checksum of config is saved
97 checksumfn
= "checksum"
100 self
.directory
= "/media/backup"
101 self
.format
= self
.formats
[0]
102 self
.epochkeeps
= { k
: 0 for k
in RealEpoch
.keys() }
103 self
.epochmodes
= { k
: "full" for k
in RealEpoch
.keys() }
104 self
.exclpatterns
= []
107 self
.lastchecksum
= None
110 return "[directory: " + self
.directory
+ \
111 ", format: " + self
.format
+ \
112 ", keeps: " + str(self
.epochkeeps
) + \
113 ", modes: " + str(self
.epochmodes
) + \
114 ", exclpatterns: " + str(self
.exclpatterns
) + \
115 ", sets: " + str([str(s
) for s
in self
.sets
]) + "]"
117 def read(self
, filename
):
118 """Read configuration from file"""
120 if not os
.path
.isfile(filename
):
121 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
123 config
= configparser
.RawConfigParser()
124 config
.read(filename
)
126 for reqsec
in ["destination"]:
127 if not config
.has_section(reqsec
):
128 raise Config
.ReadError("Section '" + reqsec
+ "' is missing.")
130 self
.directory
= config
.get("destination", "directory")
131 if not os
.path
.isdir(self
.directory
):
132 raise Config
.ReadError("Directory '{0}' does not exist.".format(self
.directory
))
134 self
.format
= config
.get("destination", "format")
135 if not self
.format
in Config
.formats
:
136 raise Config
.ReadError("Invalid 'format' given.")
139 if config
.has_section("history"):
140 for opt
in config
.options("history"):
141 if opt
.startswith("keep"):
143 if not epoch
in RealEpoch
.keys():
144 raise Config
.ReadError("Invalid option 'keep" + epoch
+ "'.")
146 self
.epochkeeps
[epoch
] = int(config
.getint("history", opt
))
148 raise Config
.ReadError("Invalid integer given for '" + opt
+ "'.")
149 elif opt
.startswith("mode"):
151 if not epoch
in RealEpoch
.keys():
152 raise Config
.ReadError("Invalid option 'mode" + epoch
+ "'.")
153 self
.epochmodes
[epoch
] = config
.get("history", opt
)
154 if not self
.epochmodes
[epoch
] in Mode
:
155 raise Config
.ReadError("Invalid mode given.")
157 raise Config
.ReadError("Invalid option '" + opt
+ "'.")
159 if config
.has_section("input"):
160 for opt
in config
.options("input"):
161 if opt
.startswith("exclude"):
162 self
.exclpatterns
+= [ config
.get("input", opt
) ]
164 raise Config
.ReadError("Invalid option '" + opt
+ "'.")
166 for sec
in config
.sections():
167 if sec
in ["destination", "history", "input"]:
169 elif sec
.startswith("set "):
170 name
= sec
[4:].strip()
173 for opt
in config
.options(sec
):
174 if not opt
.startswith("dir"):
175 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
177 dirs
+= [config
.get(sec
, opt
)]
178 self
.sets
+= [Config
.FileSet(name
, dirs
)]
180 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
182 # Compute checksum of config file
184 f
= open(filename
, 'rb')
187 self
.checksum
= m
.hexdigest()
192 f
= open(os
.path
.join(self
.directory
, self
.checksumfn
), 'r')
193 self
.lastchecksum
= f
.read().strip()
196 self
.lastchecksum
= None
200 """List and create backups"""
202 def __init__(self
, conffn
):
204 self
.conf
.read(conffn
)
207 def listAllDirs(self
):
208 """List all dirs in destination directory"""
211 basedir
= self
.conf
.directory
212 dirs
= os
.listdir(basedir
)
214 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
217 def listOldBackups(self
):
218 """Returns a list of old backups."""
222 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
223 backups
+= [ Backup
.fromDirName(entry
) ]
228 def getDesiredEpoch(self
, backups
, now
):
229 """Get desired epoch based on self.configuration and list of old backups"""
231 # Find the longest epoch for which we would like the make a backup
232 latest
= datetime
.datetime(1900, 1, 1)
233 for timespan
, e
in reversed(sorted( [ (Epoch
[e
], e
) for e
in RealEpoch
] )):
234 # We make backups of that epoch
235 if self
.conf
.epochkeeps
[e
] == 0:
238 # Get backups of that epoch
239 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
240 key
=lambda b
: b
.date
))
242 # If there are any, determine the latest
244 latest
= max(latest
, byepoch
[-1].date
)
246 # the latest backup is too old
247 if now
-latest
> timespan
:
250 # No backup is to be made
255 def backupFileSet(self
, fileset
, targetdir
, since
=None):
256 """Create an archive for given fileset at given target directory."""
258 logfile
= logging
.getLogger('backuplog')
259 logfile
.info("Running file set: " + fileset
.name
)
262 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
266 # Add the since date, if given
268 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
270 # Add the exclude patterns
271 for pat
in self
.conf
.exclpatterns
:
272 taropts
+= ["--exclude", pat
]
274 # Adding directories to backup
275 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
277 # Launch the tar process
278 tarargs
= [tarpath
] + ["-cpvaf", fsfn
] + taropts
279 logfile
.debug("tar call: " + " ".join(tarargs
))
280 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
281 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
283 # Change tarp's stdout and stderr to non-blocking
284 for s
in [tarp
.stdout
, tarp
.stderr
]:
286 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
287 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
289 # Read stdout and stderr of tarp
291 while tarp
.poll() == None:
292 rd
,wr
,ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
293 if tarp
.stdout
in rd
:
294 logging
.debug( tarp
.stdout
.readline()[:-1].decode() )
295 if tarp
.stderr
in rd
:
296 errmsg
+= tarp
.stderr
.read()
298 # Get the remainging output of tarp
299 for l
in tarp
.stdout
.readlines():
300 logging
.debug(l
.decode().rstrip())
301 errmsg
+= tarp
.stderr
.read()
303 # Get return code of tarp
306 for l
in errmsg
.decode().split("\n"):
308 logfile
.error(tarpath
+ " returned with exit status " + str(rett
) + ".")
311 def backup(self
, epoch
=None, mode
=None):
312 """Make a new backup, if necessary. If epoch is None then determine
313 desired epoch automatically. Use given epoch otherwise. If mode is None
314 then use mode for given epoch. Use given mode otherwise."""
316 now
= datetime
.datetime
.now()
317 oldbackups
= self
.listOldBackups()
319 # Get epoch of backup
321 epoch
= self
.getDesiredEpoch(oldbackups
, now
)
323 logging
.info("No backup planned.")
328 mode
= self
.conf
.epochmodes
[epoch
]
329 logging
.info("Making a backup. Epoch: " + epoch
+ ", mode: " + mode
)
331 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
333 # No old full backups existing
334 if mode
!= "full" and len(oldfullbackups
)==0:
335 logging
.info("No full backups existing. Making a full backup.")
337 # Checksum changed -> self.config file changed
338 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
339 logging
.warning("Full backup recommended as config file has changed.")
342 # If we have a full backup, we backup everything
345 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
347 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
350 logging
.debug("Making backup relative to " + since
.ctime())
352 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
356 # Create new target directory
357 basedir
= self
.conf
.directory
358 dirname
= Backup
.getDirName(now
, epoch
, mode
)
359 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
360 targetdir
= os
.path
.join(basedir
, tmpdirname
)
361 os
.mkdir( targetdir
)
365 logfile
= logging
.getLogger("backuplog")
366 fil
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
367 fil
.setLevel(logging
.DEBUG
)
368 logfile
.addHandler(fil
)
370 logfile
.info("Started: " + now
.ctime())
372 # Backup all file sets
373 for s
in self
.conf
.sets
:
374 self
.backupFileSet(s
, targetdir
, since
)
376 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
378 # Rename backup directory to final name
379 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
381 # We made a full backup -- recall checksum of config
383 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
384 f
.write( self
.conf
.checksum
)
390 """Prune old backup files"""
392 allDirs
= sorted(self
.listAllDirs())
393 # Collect all directories not matching backup name
394 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
396 # Get all directories which are kept
397 backups
= self
.listOldBackups()
399 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
400 key
=lambda b
: b
.date
, reverse
=True)) for e
in RealEpoch
}
402 keep
= self
.conf
.epochkeeps
[e
]
403 old
= byepoch
[e
][keep
:]
404 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
407 logging
.info("List of stale/outdated entries:")
415 if Backup
.isBackupDir(d
):
416 msg
+= Backup
.fromDirName(d
).colAlignedString()
422 # Check that dirs to be removed is in list of all dirs
424 assert( d
in allDirs
)
426 if len(removeDirs
) == 0:
427 logging
.info("No stale/outdated entries to remove.")
430 basedir
= self
.conf
.directory
431 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
435 shutil
.rmtree(os
.path
.join(basedir
, d
))
437 logging
.error("Error when removing '%s': %s" % (d
,e
.strerror
) )
440 def ask_user_yesno(self
, question
):
441 if LogConf
.con
.level
<= logging
.INFO
:
442 return input(question
)
448 """Print --help text"""
450 print("shbackup - a simple backup solution.")
453 print(" " + sys
.argv
[0] + " {options} [cmd]")
454 print(" " + sys
.argv
[0] + " --help")
457 print(" backup make a new backup, if necessary")
458 print(" list list all backups (default)")
459 print(" prune prune outdated/old backups")
462 print(" -h, --help print this usage text")
463 print(" -c, --conf <configfile> use given configuration file")
464 print(" default: /etc/shbackup.conf")
465 print(" -e, --epoch <epoch> force to create backup for given epoch:")
466 print(" year, month, week, day, hour, sporadic")
467 print(" -m, --mode <mode> override mode: full, diff, or incr")
468 print(" -v, --verbose be more verbose and interact with user")
469 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
470 print(" error, warning, info, debug")
471 print(" -V, --version print version info")
476 """Encapsulates logging configuration"""
478 con
= logging
.StreamHandler(sys
.stderr
)
482 """Setup logging system"""
483 conlog
= logging
.getLogger()
484 conlog
.setLevel(logging
.DEBUG
)
486 cls
.con
.setLevel(logging
.WARNING
)
487 conlog
.addHandler(cls
.con
)
489 fillog
= logging
.getLogger("backuplog")
490 fillog
.setLevel(logging
.DEBUG
)
493 if __name__
== "__main__":
497 conffn
= "/etc/shbackup.conf"
503 while i
< len(sys
.argv
)-1:
507 if opt
in ["-h", "--help"]:
511 elif opt
in ["-c", "--conf"]:
515 elif opt
in ["-V", "--version"]:
516 print("shbackup " + __version__
)
519 elif opt
in ["-v", "--verbose"]:
520 LogConf
.con
.setLevel(logging
.INFO
)
522 elif opt
in ["--verbosity"]:
525 numlevel
= getattr(logging
, level
.upper(), None)
526 if not isinstance(numlevel
, int):
527 raise ValueError('Invalid verbosity level: %s' % level
)
528 LogConf
.con
.setLevel(numlevel
)
530 elif opt
in ["-m", "--mode"]:
534 logging
.error("Unknown mode '" + mode
+ "'.")
537 elif opt
in ["-e", "--epoch"]:
540 if not epoch
in Epoch
:
541 logging
.error("Unknown epoch '" + epoch
+ "'.")
545 elif opt
in ["backup", "list", "prune"]:
549 logging
.error("Unknown option: " + opt
)
553 man
= BackupManager(conffn
)
556 man
.backup(epoch
, mode
)
559 for b
in sorted(man
.listOldBackups(), key
=lambda b
: b
.date
):
560 print(b
.colAlignedString())
565 except (Config
.ReadError
, configparser
.DuplicateOptionError
) as e
:
566 logging
.error("Error reading config file: " + e
.message
)