977fb5cd35403ccdf4fc41cc172369fa598d0b2d
2 """Stefan Huber's simplistic backup solution."""
5 __author__
= "Stefan Huber"
11 import subprocess
, fcntl
, select
16 Mode
= ["full", "incr", "diff"]
19 "hour" : datetime
.timedelta(0, 3600), \
20 "day" : datetime
.timedelta(1), \
21 "week" : datetime
.timedelta(7), \
22 "month" : datetime
.timedelta(30), \
23 "year" : datetime
.timedelta(365) }
25 Epoch
= dict(RealEpoch
, **{ \
26 "sporadic" : datetime
.timedelta(0,0) \
31 """A single backup has a date, an epoch and a mode."""
33 def __init__(self
, date
, epoch
, mode
):
39 def fromDirName(dirname
):
40 [strdate
, strtime
, epoch
, mode
] = dirname
.split("-")
42 if not epoch
in Epoch
.keys():
43 raise ValueError("Invalid epoch: " + epoch
)
46 raise ValueError("Invalid mode: " + mode
)
48 date
= datetime
.datetime(int(strdate
[0:4]),
49 int(strdate
[4:6]), int(strdate
[6:8]),\
50 int(strtime
[0:2]), int(strtime
[2:4]))
52 return Backup(date
, epoch
, mode
)
55 return "[date: " + self
.date
.ctime() + \
56 ", epoch: " + self
.epoch
+ \
57 ", mode: " + self
.mode
+ "]"
59 def colAlignedString(self
):
60 return "%16s %8s %4s" % ( \
61 self
.date
.strftime("%Y-%m-%d %H:%M"), self
.epoch
, self
.mode
)
64 def getDirName(date
, epoch
, mode
):
65 """Get directory name of backup by given properties."""
66 return date
.strftime("%Y%m%d-%H%M") + "-" + epoch
+ "-" + mode
69 def isBackupDir(dirname
):
70 """Is directory a backup directory?"""
71 p
= re
.compile(r
'^\d\d\d\d\d\d\d\d-\d\d\d\d-\w+-\w+$')
72 return p
.match(dirname
)
77 """Encapsules the configuration for the backup program."""
79 class ReadError(RuntimeError):
80 """An exception raised when reading configurations."""
81 def __init__(self
, value
):
86 """A fileset has a name and a list of directories."""
87 def __init__(self
, name
, dirs
):
92 return "[name: " + self
.name
+ ", dirs: " + str(self
.dirs
) + "]"
94 formats
= ["tar", "tar.gz", "tar.bz2", "tar.xz" ]
96 # Filename where checksum of config is saved
97 checksumfn
= "checksum"
100 self
.directory
= "/media/backup"
101 self
.format
= self
.formats
[0]
102 self
.epochkeeps
= { k
: 0 for k
in RealEpoch
.keys() }
103 self
.epochmodes
= { k
: "full" for k
in RealEpoch
.keys() }
104 self
.exclpatterns
= []
107 self
.lastchecksum
= None
110 return "[directory: " + self
.directory
+ \
111 ", format: " + self
.format
+ \
112 ", keeps: " + str(self
.epochkeeps
) + \
113 ", modes: " + str(self
.epochmodes
) + \
114 ", exclpatterns: " + str(self
.exclpatterns
) + \
115 ", sets: " + str([str(s
) for s
in self
.sets
]) + "]"
117 def read(self
, filename
):
118 """Read configuration from file"""
120 if not os
.path
.isfile(filename
):
121 raise Config
.ReadError("Cannot read config file '" + filename
+ "'.")
123 config
= configparser
.RawConfigParser()
124 config
.read(filename
)
126 for reqsec
in ["destination"]:
127 if not config
.has_section(reqsec
):
128 raise Config
.ReadError("Section '" + reqsec
+ "' is missing.")
130 self
.directory
= config
.get("destination", "directory")
131 if not os
.path
.isdir(self
.directory
):
132 raise Config
.ReadError("Directory '{0}' does not exist.".format(self
.directory
))
134 self
.format
= config
.get("destination", "format")
135 if not self
.format
in Config
.formats
:
136 raise Config
.ReadError("Invalid 'format' given.")
139 if config
.has_section("history"):
140 for opt
in config
.options("history"):
141 if opt
.startswith("keep"):
143 if not epoch
in RealEpoch
.keys():
144 raise Config
.ReadError("Invalid option 'keep" + epoch
+ "'.")
146 self
.epochkeeps
[epoch
] = int(config
.getint("history", opt
))
148 raise Config
.ReadError("Invalid integer given for '" + opt
+ "'.")
149 elif opt
.startswith("mode"):
151 if not epoch
in RealEpoch
.keys():
152 raise Config
.ReadError("Invalid option 'mode" + epoch
+ "'.")
153 self
.epochmodes
[epoch
] = config
.get("history", opt
)
154 if not self
.epochmodes
[epoch
] in Mode
:
155 raise Config
.ReadError("Invalid mode given.")
157 raise Config
.ReadError("Invalid option '" + opt
+ "'.")
159 if config
.has_section("input"):
160 for opt
in config
.options("input"):
161 if opt
.startswith("exclude"):
162 self
.exclpatterns
+= [ config
.get("input", opt
) ]
164 raise Config
.ReadError("Invalid option '" + opt
+ "'.")
166 for sec
in config
.sections():
167 if sec
in ["destination", "history", "input"]:
169 elif sec
.startswith("set "):
170 name
= sec
[4:].strip()
173 for opt
in config
.options(sec
):
174 if not opt
.startswith("dir"):
175 raise Config
.ReadError("Unknown option '" + opt
+ "'.")
177 dirs
+= [config
.get(sec
, opt
)]
178 self
.sets
+= [Config
.FileSet(name
, dirs
)]
180 raise Config
.ReadError("Unknown section '" + sec
+ "'.")
182 # Compute checksum of config file
184 f
= open(filename
, 'rb')
187 self
.checksum
= m
.hexdigest()
192 f
= open(os
.path
.join(self
.directory
, self
.checksumfn
), 'r')
193 self
.lastchecksum
= f
.read().strip()
196 self
.lastchecksum
= None
200 """List and create backups"""
202 def __init__(self
, conffn
):
204 self
.conf
.read(conffn
)
207 def listAllDirs(self
):
208 """List all dirs in destination directory"""
211 basedir
= self
.conf
.directory
212 dirs
= os
.listdir(basedir
)
214 return [ d
for d
in dirs
if os
.path
.isdir(os
.path
.join(basedir
, d
)) ]
217 def listOldBackups(self
):
218 """Returns a list of old backups."""
222 for entry
in [ b
for b
in self
.listAllDirs() if Backup
.isBackupDir(b
) ]:
223 backups
+= [ Backup
.fromDirName(entry
) ]
228 def getDesiredEpoch(self
, backups
, now
):
229 """Get desired epoch based on self.configuration and list of old backups"""
231 # Find the longest epoch for which we would like the make a backup
232 latest
= datetime
.datetime(1900, 1, 1)
233 for timespan
, e
in reversed(sorted( [ (Epoch
[e
], e
) for e
in RealEpoch
] )):
234 # We make backups of that epoch
235 if self
.conf
.epochkeeps
[e
] == 0:
238 # Get backups of that epoch
239 byepoch
= list(sorted( [ b
for b
in backups
if b
.epoch
==e
], \
240 key
=lambda b
: b
.date
))
242 # If there are any, determine the latest
244 latest
= max(latest
, byepoch
[-1].date
)
246 # the latest backup is too old
247 if now
-latest
> timespan
:
250 # No backup is to be made
255 def backupFileSet(self
, fileset
, targetdir
, since
=None):
256 """Create an archive for given fileset at given target directory."""
258 logfile
= logging
.getLogger('backuplog')
259 logfile
.info("Running file set: " + fileset
.name
)
262 fsfn
= os
.path
.join(targetdir
, fileset
.name
) + "." + self
.conf
.format
266 # Add the since date, if given
268 taropts
+= ["-N", since
.strftime("%Y-%m-%d %H:%M:%S")]
270 # Add the exclude patterns
271 for pat
in self
.conf
.exclpatterns
:
272 taropts
+= ["--exclude", pat
]
274 # Adding directories to backup
275 taropts
+= ["-C", "/"] + [ "./" + d
.lstrip("/") for d
in fileset
.dirs
]
277 # Launch the tar process
278 tarargs
= [tarpath
] + ["-cpvaf", fsfn
] + taropts
279 logfile
.debug("tar call: " + " ".join(tarargs
))
280 tarp
= subprocess
.Popen( tarargs
, bufsize
=-1, \
281 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
283 # Change tarp's stdout and stderr to non-blocking
284 for s
in [tarp
.stdout
, tarp
.stderr
]:
286 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
287 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
289 def readlineNonBlocking(stream
):
290 """Read a line nonblocking. Returns b'' if nothing read."""
292 return stream
.readline()
298 # Read stdout and stderr of tarp
300 while tarp
.poll() == None:
301 rd
,wr
,ex
= select
.select([tarp
.stdout
, tarp
.stderr
], [], [], 0.05)
303 if tarp
.stdout
in rd
:
304 l
= readlineNonBlocking(tarp
.stdout
)
306 logging
.debug(l
[:-1].decode())
308 if tarp
.stderr
in rd
:
309 errmsg
+= readlineNonBlocking(tarp
.stderr
)
312 # Get the remainging output of tarp
313 for l
in tarp
.stdout
.readlines():
314 logging
.debug(l
.decode().rstrip())
315 errmsg
+= tarp
.stderr
.read()
317 # Get return code of tarp
320 for l
in errmsg
.decode().split("\n"):
322 logfile
.error(tarpath
+ " returned with exit status " + str(rett
) + ".")
325 def backup(self
, epoch
=None, mode
=None):
326 """Make a new backup, if necessary. If epoch is None then determine
327 desired epoch automatically. Use given epoch otherwise. If mode is None
328 then use mode for given epoch. Use given mode otherwise."""
330 now
= datetime
.datetime
.now()
331 oldbackups
= self
.listOldBackups()
333 # Get epoch of backup
335 epoch
= self
.getDesiredEpoch(oldbackups
, now
)
337 logging
.info("No backup planned.")
342 mode
= self
.conf
.epochmodes
[epoch
]
343 logging
.info("Making a backup. Epoch: " + epoch
+ ", mode: " + mode
)
345 oldfullbackups
= [ b
for b
in oldbackups
if b
.mode
== "full" ]
347 # No old full backups existing
348 if mode
!= "full" and len(oldfullbackups
)==0:
349 logging
.info("No full backups existing. Making a full backup.")
351 # Checksum changed -> self.config file changed
352 if self
.conf
.checksum
!= self
.conf
.lastchecksum
and mode
!= "full":
353 logging
.warning("Full backup recommended as config file has changed.")
356 # If we have a full backup, we backup everything
359 since
= sorted(oldfullbackups
, key
=lambda b
: b
.date
)[-1].date
361 since
= sorted(oldbackups
, key
=lambda b
: b
.date
)[-1].date
364 logging
.debug("Making backup relative to " + since
.ctime())
366 yesno
= self
.ask_user_yesno("Proceed? [Y, n] ")
370 # Create new target directory
371 basedir
= self
.conf
.directory
372 dirname
= Backup
.getDirName(now
, epoch
, mode
)
373 tmpdirname
= dirname
+ ("-%x" % (random
.random()*2e16
) )
374 targetdir
= os
.path
.join(basedir
, tmpdirname
)
375 os
.mkdir( targetdir
)
379 logfile
= logging
.getLogger("backuplog")
380 fil
= logging
.FileHandler( os
.path
.join(targetdir
, "log") )
381 fil
.setLevel(logging
.DEBUG
)
382 logfile
.addHandler(fil
)
384 logfile
.info("Started: " + now
.ctime())
386 # Backup all file sets
387 for s
in self
.conf
.sets
:
388 self
.backupFileSet(s
, targetdir
, since
)
390 logfile
.info("Stopped: " + datetime
.datetime
.now().ctime())
392 # Rename backup directory to final name
393 os
.rename( targetdir
, os
.path
.join(basedir
, dirname
) )
395 # We made a full backup -- recall checksum of config
397 f
= open( os
.path
.join(basedir
, self
.conf
.checksumfn
), "w")
398 f
.write( self
.conf
.checksum
)
404 """Prune old backup files"""
406 allDirs
= sorted(self
.listAllDirs())
407 # Collect all directories not matching backup name
408 removeDirs
= [ d
for d
in allDirs
if not Backup
.isBackupDir(d
) ]
410 # Get all directories which are kept
411 backups
= self
.listOldBackups()
413 byepoch
= { e
: list(sorted( [ b
for b
in backups
if b
.epoch
== e
], \
414 key
=lambda b
: b
.date
, reverse
=True)) for e
in RealEpoch
}
416 keep
= self
.conf
.epochkeeps
[e
]
417 old
= byepoch
[e
][keep
:]
418 removeDirs
+= [ Backup
.getDirName(b
.date
, b
.epoch
, b
.mode
) for b
in old
]
421 logging
.info("List of stale/outdated entries:")
429 if Backup
.isBackupDir(d
):
430 msg
+= Backup
.fromDirName(d
).colAlignedString()
436 # Check that dirs to be removed is in list of all dirs
438 assert( d
in allDirs
)
440 if len(removeDirs
) == 0:
441 logging
.info("No stale/outdated entries to remove.")
444 basedir
= self
.conf
.directory
445 yesno
= self
.ask_user_yesno("Remove entries marked by '*'? [y, N] ")
449 shutil
.rmtree(os
.path
.join(basedir
, d
))
451 logging
.error("Error when removing '%s': %s" % (d
,e
.strerror
) )
454 def ask_user_yesno(self
, question
):
455 if LogConf
.con
.level
<= logging
.INFO
:
456 return input(question
)
462 """Print --help text"""
464 print("shbackup - a simple backup solution.")
467 print(" " + sys
.argv
[0] + " {options} [cmd]")
468 print(" " + sys
.argv
[0] + " --help")
471 print(" backup make a new backup, if necessary")
472 print(" list list all backups (default)")
473 print(" prune prune outdated/old backups")
476 print(" -h, --help print this usage text")
477 print(" -c, --conf <configfile> use given configuration file")
478 print(" default: /etc/shbackup.conf")
479 print(" -e, --epoch <epoch> force to create backup for given epoch:")
480 print(" year, month, week, day, hour, sporadic")
481 print(" -m, --mode <mode> override mode: full, diff, or incr")
482 print(" -v, --verbose be more verbose and interact with user")
483 print(" --verbosity LEVEL set verbosity to LEVEL, which can be")
484 print(" error, warning, info, debug")
485 print(" -V, --version print version info")
490 """Encapsulates logging configuration"""
492 con
= logging
.StreamHandler(sys
.stderr
)
496 """Setup logging system"""
497 conlog
= logging
.getLogger()
498 conlog
.setLevel(logging
.DEBUG
)
500 cls
.con
.setLevel(logging
.WARNING
)
501 conlog
.addHandler(cls
.con
)
503 fillog
= logging
.getLogger("backuplog")
504 fillog
.setLevel(logging
.DEBUG
)
507 if __name__
== "__main__":
511 conffn
= "/etc/shbackup.conf"
517 while i
< len(sys
.argv
)-1:
521 if opt
in ["-h", "--help"]:
525 elif opt
in ["-c", "--conf"]:
529 elif opt
in ["-V", "--version"]:
530 print("shbackup " + __version__
)
533 elif opt
in ["-v", "--verbose"]:
534 LogConf
.con
.setLevel(logging
.INFO
)
536 elif opt
in ["--verbosity"]:
539 numlevel
= getattr(logging
, level
.upper(), None)
540 if not isinstance(numlevel
, int):
541 raise ValueError('Invalid verbosity level: %s' % level
)
542 LogConf
.con
.setLevel(numlevel
)
544 elif opt
in ["-m", "--mode"]:
548 logging
.error("Unknown mode '" + mode
+ "'.")
551 elif opt
in ["-e", "--epoch"]:
554 if not epoch
in Epoch
:
555 logging
.error("Unknown epoch '" + epoch
+ "'.")
559 elif opt
in ["backup", "list", "prune"]:
563 logging
.error("Unknown option: " + opt
)
567 man
= BackupManager(conffn
)
570 man
.backup(epoch
, mode
)
573 for b
in sorted(man
.listOldBackups(), key
=lambda b
: b
.date
):
574 print(b
.colAlignedString())
579 except (Config
.ReadError
, configparser
.DuplicateOptionError
) as e
:
580 logging
.error("Error reading config file: " + e
.message
)