]>
git.sthu.org Git - sitarba.git/blob - shbackup
2 """Stefan Huber's simplistic backup solution."""
5 __author__
= "Stefan Huber"
16 Mode
= ["full", "incr", "diff"]
19 "hour" : datetime
.timedelta(0, 3600), \
20 "day" : datetime
.timedelta(1), \
21 "week" : datetime
.timedelta(7), \
22 "month" : datetime
.timedelta(30), \
23 "year" : datetime
.timedelta(365) }
25 Epoch
= dict(RealEpoch
, **{ \
26 "sporadic" : datetime
.timedelta(0,0) \
31 """A single backup has a date, an epoch and a mode."""
33 def __init__(self
, date
, epoch
, mode
):
39 def fromDirName(dirname
):
40 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
42 if not epoch
in Epoch
.keys():
43 raise ValueError("Invalid epoch: " + epoch
)
46 raise ValueError("Invalid mode: " + mode
)
48 date
= datetime
.datetime(int(strdate
[0:4]),
49 int(strdate
[4:6]), int(strdate
[6:8]),\
50 int(strtime
[0:2]), int(strtime
[2:4]))
52 return Backup(date
, epoch
, mode
)
55 return "[date: " + self
.date
.ctime() + \
56 ", epoch: " + self
.epoch
+ \
57 ", mode: " + self
.mode
+ "]"
59 def colAlignedString(self
):
60 return "%16s %8s %4s" % ( \
61 self
.date
.strftime("%Y-%m-%d %H:%M"), self
.epoch
, self
.mode
)
64 def getDirName(date
, epoch
, mode
):
65 """Get directory name of backup by given properties."""
66 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
69 def isBackupDir(dirname
):
70 """Is directory a backup directory?"""
71 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
72 return p
.match(dirname
)
77 """Encapsules the configuration for the backup program."""
79 class ReadError(RuntimeError):
80 """An exception raised when reading configurations."""
81 def __init__(self
, value
):
86 """A fileset has a name and a list of directories."""
87 def __init__(self
, name
, dirs
):
92 return "[name: " + self
.name
+ ", dirs: " + str(self
.dirs
) + "]"
94 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
96 # Filename where checksum of config is saved
97 checksumfn
= "checksum"
100 self
.directory
= "/media/backup"
101 self
.format
= self
.formats
[0]
102 self
.epochkeeps
= { k
: 0 for k
in RealEpoch
.keys() }
103 self
.epochmodes
= { k
: "full" for k
in RealEpoch
.keys() }
104 self
.exclpatterns
= []
107 self
.lastchecksum
= None
110 return "[directory: " + self
.directory
+ \
111 ", format: " + self
.format
+ \
112 ", keeps: " + str(self
.epochkeeps
) + \
113 ", modes: " + str(self
.epochmodes
) + \
114 ", exclpatterns: " + str(self
.exclpatterns
) + \
115 ", sets: " + str([str(s
) for s
in self
.sets
]) + "]"
117 def read(self
, filename
):
118 """Read configuration from file"""
120 if not os
.path
.isfile(filename
):
121 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
123 config
= configparser
.RawConfigParser()
124 config
.read(filename
)
126 for reqsec
in ["destination"]:
127 if not config
.has_section(reqsec
):
128 raise Config
.ReadError("Section '" + reqsec
+ "' is missing.")
130 self
.directory
= config
.get("destination", "directory")
131 if not os
.path
.isdir(self
.directory
):
132 raise Config
.ReadError("Directory '{0}' does not exist.".format(self
.directory
))
134 self
.format
= config
.get("destination", "format")
135 if not self
.format
in Config
.formats
:
136 raise Config
.ReadError("Invalid 'format' given.")
139 if config
.has_section("history"):
140 for opt
in config
.options("history"):
141 if opt
.startswith("keep"):
143 if not epoch
in RealEpoch
.keys():
144 raise Config
.ReadError("Invalid option 'keep" + epoch
+ "'.")
146 self
.epochkeeps
[epoch
] = int(config
.getint("history", opt
))
148 raise Config
.ReadError("Invalid integer given for '" + opt
+ "'.")
149 elif opt
.startswith("mode"):
151 if not epoch
in RealEpoch
.keys():
152 raise Config
.ReadError("Invalid option 'mode" + epoch
+ "'.")
153 self
.epochmodes
[epoch
] = config
.get("history", opt
)
154 if not self
.epochmodes
[epoch
] in Mode
:
155 raise Config
.ReadError("Invalid mode given.")
157 raise Config
.ReadError("Invalid option '" + opt
+ "'.")
159 if config
.has_section("input"):
160 for opt
in config
.options("input"):
161 if opt
.startswith("exclude"):
162 self
.exclpatterns
+= [ config
.get("input", opt
) ]
164 raise Config
.ReadError("Invalid option '" + opt
+ "'.")
166 for sec
in config
.sections():
167 if sec
in ["destination", "history", "input"]:
169 elif sec
.startswith("set "):
170 name
= sec
[4:].strip()
173 for opt
in config
.options(sec
):
174 if not opt
.startswith("dir"):
175 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
177 dirs
+= [config
.get(sec
, opt
)]
178 self
.sets
+= [Config
.FileSet(name
, dirs
)]
180 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
182 # Compute checksum of config file
184 f
= open(filename
, 'rb')
187 self
.checksum
= m
.hexdigest()
192 f
= open(os
.path
.join(self
.directory
, self
.checksumfn
), 'r')
193 self
.lastchecksum
= f
.read().strip()
196 self
.lastchecksum
= None
200 """List and create backups"""
202 def __init__(self
, conffn
):
204 self
.conf
.read(conffn
)
207 def listAllDirs(self
):
208 """List all dirs in destination directory"""
211 basedir
= self
.conf
.directory
212 dirs
= os
.listdir(basedir
)
214 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
217 def listOldBackups(self
):
218 """Returns a list of old backups."""
222 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
223 backups
+= [ Backup
.fromDirName(entry
) ]
228 def getDesiredEpoch(self
, backups
, now
):
229 """Get desired epoch based on self.configuration and list of old backups"""
231 # Find the longest epoch for which we would like the make a backup
232 latest
= datetime
.datetime(1900, 1, 1)
233 for timespan
, e
in reversed(sorted( [ (Epoch
[e
], e
) for e
in RealEpoch
] )):
234 # We make backups of that epoch
235 if self
.conf
.epochkeeps
[e
] == 0:
238 # Get backups of that epoch
239 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
240 key
=lambda b
: b
.date
))
242 # If there are any, determine the latest
244 latest
= max(latest
, byepoch
[-1].date
)
246 # the latest backup is too old
247 if now
-latest
> timespan
:
250 # No backup is to be made
255 def backupFileSet(self
, fileset
, targetdir
, since
=None):
256 """Create an archive for given fileset at given target directory."""
258 logger
= logging
.getLogger('backup')
260 logger
.info("Running file set: " + fileset
.name
)
262 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
267 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
269 for pat
in self
.conf
.exclpatterns
:
270 taropts
+= ["--exclude", pat
]
272 tarargs
= [tarpath
] + taropts
+ ["-f", fsfn
] + fileset
.dirs
273 logger
.debug("tar call: " + " ".join(tarargs
))
274 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
275 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
277 # Output stdout of tar
278 while tarp
.poll() == None:
279 l
= tarp
.stdout
.readline()
281 logging
.debug(l
.decode().rstrip())
283 # Output remaining output of tar
284 for l
in tarp
.stdout
.readlines():
285 logging
.debug(l
.decode().rstrip())
289 for l
in tarp
.stderr
.readlines():
290 logger
.error( l
.decode().strip().rstrip() )
291 sys
.stderr
.write( tarp
.stderr
.read().decode() )
292 logger
.error(tarpath
+ " returned with exit status " + str(rett
) + ".")
295 def backup(self
, epoch
=None, mode
=None):
296 """Make a new backup, if necessary. If epoch is None then determine
297 desired epoch automatically. Use given epoch otherwise. If mode is None
298 then use mode for given epoch. Use given mode otherwise."""
300 now
= datetime
.datetime
.now()
301 oldbackups
= self
.listOldBackups()
303 # Get epoch of backup
305 epoch
= self
.getDesiredEpoch(oldbackups
, now
)
307 logging
.info("No backup planned.")
312 mode
= self
.conf
.epochmodes
[epoch
]
313 logging
.info("Making a backup. Epoch: " + epoch
+ ", mode: " + mode
)
315 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
317 # No old full backups existing
318 if mode
!= "full" and len(oldfullbackups
)==0:
319 logging
.info("No full backups existing. Making a full backup.")
321 # Checksum changed -> self.config file changed
322 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
323 logging
.warning("Full backup recommended as config file has changed.")
326 # If we have a full backup, we backup everything
329 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
331 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
334 logging
.debug("Making backup relative to " + since
.ctime())
336 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
340 # Create new target directory
341 basedir
= self
.conf
.directory
342 dirname
= Backup
.getDirName(now
, epoch
, mode
)
343 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
344 targetdir
= os
.path
.join(basedir
, tmpdirname
)
345 os
.mkdir( targetdir
)
348 logger
= logging
.getLogger('backup')
349 ch
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
350 ch
.setLevel(logging
.INFO
)
351 logger
.addHandler(ch
)
352 logger
.info("Started: " + now
.ctime())
354 # Backup all file sets
355 for s
in self
.conf
.sets
:
356 self
.backupFileSet(s
, targetdir
, since
)
358 logger
.info("Stopped: " + datetime
.datetime
.now().ctime())
360 # Rename backup directory to final name
361 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
363 # We made a full backup -- recall checksum of config
365 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
366 f
.write( self
.conf
.checksum
)
372 """Prune old backup files"""
374 allDirs
= sorted(self
.listAllDirs())
375 # Collect all directories not matching backup name
376 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
378 # Get all directories which are kept
379 backups
= self
.listOldBackups()
381 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
382 key
=lambda b
: b
.date
, reverse
=True)) for e
in RealEpoch
}
384 keep
= self
.conf
.epochkeeps
[e
]
385 old
= byepoch
[e
][keep
:]
386 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
389 logging
.info("List of stale/outdated entries:")
397 if Backup
.isBackupDir(d
):
398 msg
+= Backup
.fromDirName(d
).colAlignedString()
404 # Check that dirs to be removed is in list of all dirs
406 assert( d
in allDirs
)
408 if len(removeDirs
) == 0:
409 logging
.info("No stale/outdated entries to remove.")
412 basedir
= self
.conf
.directory
413 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
416 shutil
.rmtree(os
.path
.join(basedir
, d
))
418 def ask_user_yesno(self
, question
):
419 if logging
.getLogger().isEnabledFor(logging
.INFO
):
420 return input(question
)
426 """Print --help text"""
428 print("shbackup - a simple backup solution.")
431 print(" " + sys
.argv
[0] + " {options} [cmd]")
432 print(" " + sys
.argv
[0] + " --help")
435 print(" backup make a new backup, if necessary")
436 print(" list list all backups (default)")
437 print(" prune prune outdated/old backups")
440 print(" -h, --help print this usage text")
441 print(" -c, --conf <configfile> use given configuration file")
442 print(" default: /etc/shbackup.conf")
443 print(" -e, --epoch <epoch> force to create backup for given epoch:")
444 print(" year, month, week, day, hour, sporadic")
445 print(" -m, --mode <mode> override mode: full, diff, or incr")
446 print(" -v, --verbose be more verbose and interact with user")
447 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
448 print(" warning, info, debug")
449 print(" -V, --version print version info")
452 if __name__
== "__main__":
454 logging
.basicConfig(format
='%(message)s')
455 conffn
= "/etc/shbackup.conf"
461 while i
< len(sys
.argv
)-1:
465 if opt
in ["-h", "--help"]:
469 elif opt
in ["-c", "--conf"]:
473 elif opt
in ["-V", "--version"]:
474 print("shbackup " + __version__
)
477 elif opt
in ["-v", "--verbose"]:
478 logging
.getLogger().setLevel(logging
.INFO
)
480 elif opt
in ["--verbosity"]:
483 numlevel
= getattr(logging
, level
.upper(), None)
484 if not isinstance(numlevel
, int):
485 raise ValueError('Invalid verbosity level: %s' % level
)
486 logging
.getLogger().setLevel(numlevel
)
488 elif opt
in ["-m", "--mode"]:
492 logging
.error("Unknown mode '" + mode
+ "'.")
495 elif opt
in ["-e", "--epoch"]:
498 if not epoch
in Epoch
:
499 logging
.error("Unknown epoch '" + epoch
+ "'.")
503 elif opt
in ["backup", "list", "prune"]:
507 logging
.error("Unknown option: " + opt
)
511 man
= BackupManager(conffn
)
514 man
.backup(epoch
, mode
)
517 for b
in sorted(man
.listOldBackups(), key
=lambda b
: b
.date
):
518 print(b
.colAlignedString())
523 except (Config
.ReadError
, configparser
.DuplicateOptionError
) as e
:
524 logging
.error("Error reading config file: " + e
.message
)