[Box Backup-commit] COMMIT r2136 - in box/trunk/contrib: . bbreporter

boxbackup-dev at fluffy.co.uk boxbackup-dev at fluffy.co.uk
Mon Apr 7 08:59:17 BST 2008


Author: cbkm
Date: 2008-04-07 07:59:16 +0000 (Mon, 07 Apr 2008)
New Revision: 2136

Added:
   box/trunk/contrib/bbreporter/
   box/trunk/contrib/bbreporter/bbreporter.py
Log:
First commit of bbreporter.py



Added: box/trunk/contrib/bbreporter/bbreporter.py
===================================================================
--- box/trunk/contrib/bbreporter/bbreporter.py	                        (rev 0)
+++ box/trunk/contrib/bbreporter/bbreporter.py	2008-04-07 07:59:16 UTC (rev 2136)
@@ -0,0 +1,488 @@
+#!/usr/bin/env python
+# BoxBackupReporter - Simple script to report on backups that have been 
+#                     performed using BoxBackup.
+#
+# Copyright: (C) 2007 Three A IT Limited
+# Author: Kenny Millington <kenny.millington at 3ait.co.uk>
+# Version: $Id: bbreporter.py,v 1.15 2007-11-14 11:13:38 kenny Exp $
+#
+# Credit: This script is based on the ideas of BoxReport.pl by Matt Brown of 
+#         Three A IT Support Limited.
+#
+################################################################################
+# !! Important !!
+# To make use of this script you need to run the boxbackup client with the -v
+# commandline option and set LogAllFileAccess = yes in your bbackupd.conf file.
+# 
+# Notes on lazy mode:
+# If reporting on lazy mode backups you absolutely must ensure that
+# logrotate (or similar) rotates the log files at the same rate at 
+# which you run this reporting script or you will report on the same
+# backup sessions on each execution.
+#
+# Notes on --rotate and log rotation in general: 
+# The use-case for --rotate that I imagine is that you'll add a line like the 
+# following into your syslog.conf file:-
+# 
+# local6.* -/var/log/box
+# 
+# Then specifying --rotate to this script will make it rotate the logs
+# each time you report on the backup so that you don't risk a backup session 
+# being spread across two log files (e.g. syslog and syslog.0). 
+#
+# NB: To do this you'll need to prevent logrotate/syslog from rotating your
+#     /var/log/box file. On Debian based distros you'll need to edit two files.
+#     
+#     First: /etc/cron.daily/sysklogd, find the following line and make the 
+#            the required change: 
+#            Change: for LOG in `syslogd-listfiles`
+#                To: for LOG in `syslogd-listfiles -s box`
+#                                 
+#     Second: /etc/cron.weekly/sysklogd, find the following line and make the 
+#            the required change: 
+#            Change: for LOG in `syslogd-listfiles --weekly`
+#                To: for LOG in `syslogd-listfiles --weekly -s box`
+#
+# Alternatively, if suitable just ensure the backups stop before the 
+# /etc/cron.daily/sysklogd file runs (usually 6:25am) and report on it
+# before the files get rotated. (If going for this option I'd just use
+# the main syslog file instead of creating a separate log file for box
+# backup since you know for a fact the syslog will get rotated daily.)
+#
+################################################################################
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.  
+#
+
+# If sendmail is not in one of these paths, add the path.
+SENDMAIL_PATHS = ["/usr/sbin/", "/usr/bin/", "/bin/" , "/sbin/"]
+
+# The name of the sendmail binary, you probably won't need to change this.
+SENDMAIL_BIN = "sendmail"
+
+# Number of files to rotate around
+ROTATE_COUNT = 7
+
+# Import the required libraries
+import sys, os, re, getopt, shutil, gzip
+
+class BoxBackupReporter:
+    class BoxBackupReporterError(Exception):
+        pass
+    
+    def __init__(self, config_file="/etc/box/bbackupd.conf", 
+                 log_file="/var/log/syslog", email_to=None, 
+                 email_from="report at boxbackup", rotate=False, 
+                 verbose=False, stats=False):
+        
+        # Config options
+        self.config_file = config_file
+        self.log_file = log_file
+        self.email_to = email_to
+        self.email_from = email_from
+        self.rotate_log_file = rotate
+        self.verbose_report = verbose
+        self.usage_stats = stats
+        
+        # Regex's
+        self.re_automatic_backup = re.compile(" *AutomaticBackup *= *no", re.I)
+        self.re_syslog = re.compile("(\S+) +(\S+) +([\d:]+) +(\S+) +([^:]+): +"+
+                                    "([^:]+): *(.*)") 
+        
+        # Initialise report
+        self.reset()
+        
+        
+    def reset(self):
+        # Reset report data to default values
+        self.hostname = ""
+        self.patched_files = {}
+        self.synced_files = {}
+        self.uploaded_files = {}
+        self.warnings = []
+        self.errors = []
+        self.stats = None
+        self.start_datetime = "Unknown"
+        self.end_datetime = "Unfinished"
+        self.report = "No report generated"
+    
+    def run(self):
+        try:
+            self._determine_operating_mode()
+        except IOError:
+            raise BoxBackupReporter.BoxBackupReporterError("Error: "+\
+                  "Config file \"%s\" could not be read." % self.config_file)
+            
+        try:
+            self._parse_syslog()
+        except IOError:
+            raise BoxBackupReporter.BoxBackupReporterError("Error: "+\
+                  "Log file \"%s\" could not be read." % self.log_file)
+        
+        self._parse_stats()
+        self._generate_report()
+        
+    def deliver(self):
+        # If we're not e-mailing the report then just dump it to stdout
+        # and return.
+        if self.email_to is None:
+            print self.report
+            # Now that we've delivered the report it's time to rotate the logs
+            # if we're requested to do so.
+            self._rotate_log()
+            return
+        
+        # Locate the sendmail binary
+        sendmail = self._locate_sendmail()
+        if(sendmail is None):
+            raise BoxBackupReporter.BoxBackupReporterError("Error: "+\
+                  "Could not find sendmail binary - Unable to send e-mail!")
+            
+        
+        # Set the subject based on whether we think we failed or not.
+        # (suffice it to say I consider getting an error and backing up
+        #  no files a failure or indeed not finding a start time in the logs).
+        subject = "BoxBackup Reporter (%s) - " % self.hostname
+        if self.start_datetime == "Unknown" or\
+            (len(self.patched_files)  == 0 and len(self.synced_files) == 0 and\
+             len(self.uploaded_files) == 0):
+            subject = subject + "FAILED"
+        else:
+            subject = subject + "SUCCESS"
+
+            if len(self.errors) > 0:
+                subject = subject + " (with errors)"
+            
+        # Prepare the e-mail message.
+        mail = []
+        mail.append("To: " + self.email_to)
+        mail.append("From: " + self.email_from)
+        mail.append("Subject: " + subject)
+        mail.append("")
+        mail.append(self.report)
+        
+        # Send the mail.
+        p = os.popen(sendmail + " -t", "w")
+        p.write("\r\n".join(mail))
+        p.close()
+        
+        # Now that we've delivered the report it's time to rotate the logs
+        # if we're requested to do so.
+        self._rotate_log()
+         
+    def _determine_operating_mode(self):
+        # Scan the config file and determine if we're running in lazy or 
+        # snapshot mode.
+        cfh = open(self.config_file)
+
+        for line in cfh:
+            if not line.startswith("#"):
+                if self.re_automatic_backup.match(line):
+                    self.lazy_mode = False
+                    cfh.close()
+                    return
+        
+        self.lazy_mode = True
+        cfh.close()
+    
+    def _parse_syslog(self):
+        lfh = open(self.log_file)
+        
+        for line in lfh:
+            # Only run the regex if we find a box backup entry.
+            if line.find("Box Backup") > -1 or line.find("bbackupd") > -1:
+                raw_data = self.re_syslog.findall(line)
+                try:
+                    data = raw_data[0]
+                except IndexError:
+                    # If the regex didn't match it's not a message that we're
+                    # interested in so move to the next line.
+                    continue
+                
+                # Set the hostname, it shouldn't change in a log file
+                self.hostname = data[3]
+                
+                # If we find the backup-start event then set the start_datetime.
+                if data[6].find("backup-start") > -1:
+                    # If we're not in lazy mode or the start_datetime hasn't
+                    # been set then reset the data and set it.
+                    #
+                    # If we're in lazy mode and encounter a second backup-start
+                    # we don't want to change the start_datetime likewise if 
+                    # we're not in lazy mode we do want to and we want to reset
+                    # so we only capture the most recent session.
+                    if not self.lazy_mode or self.start_datetime == "Unknown":
+                        self.reset()
+                        self.start_datetime = data[1]+" "+data[0]+ " "+data[2]
+                                              
+                # If we find the backup-finish event then set the end_datetime.
+                elif data[6].find("backup-finish") > -1:
+                    self.end_datetime = data[1] + " " + data[0] + " " + data[2]
+                
+                # Only log the events if we have our start time.
+                elif self.start_datetime != "Unknown":
+                    # We found a patch event, add the file to the patched_files.
+                    if data[5] == "Uploading patch to file":
+                        self.patched_files[data[6]] = ""
+
+                    # We found an upload event, add to uploaded files.
+                    elif data[5] == "Uploading complete file":
+                        self.uploaded_files[data[6]] = ""
+                    
+                    # We found another upload event.
+                    elif data[5] == "Uploaded file":
+                        self.uploaded_files[data[6]] = ""
+                    
+                    # We found a sync event, add the file to the synced_files.
+                    elif data[5] == "Synchronised file":
+                        self.synced_files[data[6]] = ""
+                
+                    # We found a warning, add the warning to the warnings.
+                    elif data[5] == "WARNING":
+                        self.warnings.append(data[6])
+
+                    # We found an error, add the error to the errors.
+                    elif data[5] == "ERROR":
+                        self.errors.append(data[6])
+
+        lfh.close()
+    
+    def _parse_stats(self):
+        if(not self.usage_stats):
+            return
+        
+        # Grab the stats from bbackupquery
+        sfh = os.popen("bbackupquery usage quit", "r")
+        raw_stats = sfh.read()
+        sfh.close()
+        
+        # Parse the stats 
+        stats_re = re.compile("commands.[\n ]*\n(.*)\n+", re.S)
+        stats = stats_re.findall(raw_stats)
+        
+        try:
+            self.stats = stats[0]
+        except IndexError:
+            self.stats = "Unable to retrieve usage information."
+            
+    def _generate_report(self):
+        if self.start_datetime == "Unknown":
+            self.report = "No report data has been found."
+            return
+        
+        total_files = len(self.patched_files) + len(self.uploaded_files)
+        
+        report = []
+        report.append("--------------------------------------------------")
+        report.append("Report Title  : Box Backup - Backup Statistics")
+        report.append("Report Period : %s - %s" % (self.start_datetime, 
+                                                   self.end_datetime))
+        report.append("--------------------------------------------------")
+        report.append("")
+        report.append("This is your box backup report, in summary:")
+        report.append("")
+        report.append("%d file(s) have been backed up." % total_files)
+        report.append("%d file(s) were uploaded." % len(self.uploaded_files))
+        report.append("%d file(s) were patched." % len(self.patched_files))
+        report.append("%d file(s) were synchronised." % len(self.synced_files))
+        
+        report.append("")
+        report.append("%d warning(s) occurred." % len(self.warnings))
+        report.append("%d error(s) occurred." % len(self.errors))
+        report.append("")
+        report.append("")
+        
+        # If we asked for the backup stats and they're available
+        # show them.
+        if(self.stats is not None and self.stats != ""):
+            report.append("Your backup usage information follows:")
+            report.append("")
+            report.append(self.stats)
+            report.append("")
+            report.append("")
+            
+        # List the files if we've been asked for a verbose report.
+        if(self.verbose_report):
+            if len(self.uploaded_files) > 0:
+                report.append("Uploaded Files (%d)" % len(self.uploaded_files))
+                report.append("---------------------")
+                for file in self.uploaded_files.keys():
+                    report.append(file)
+                report.append("")
+                report.append("")
+            
+            if len(self.patched_files) > 0:
+                report.append("Patched Files (%d)" % len(self.patched_files))
+                report.append("---------------------")
+                for file in self.patched_files.keys():
+                    report.append(file)
+                report.append("")
+                report.append("")
+    
+        # Always output the warnings/errors.
+        if len(self.warnings) > 0:
+            report.append("Warnings (%d)" % len(self.warnings))
+            report.append("---------------------")
+            for warning in self.warnings:
+                report.append(warning)
+            report.append("")
+            report.append("")
+        
+        if len(self.errors) > 0:
+            report.append("Errors (%d)" % len(self.errors))
+            report.append("---------------------")
+            for error in self.errors:
+                report.append(error)
+            report.append("")
+            report.append("")
+
+        self.report = "\r\n".join(report)
+
+    def _locate_sendmail(self):
+        for path in SENDMAIL_PATHS:
+            sendmail = os.path.join(path, SENDMAIL_BIN)
+            if os.path.isfile(sendmail):
+                return sendmail
+
+        return None
+    
+    def _rotate_log(self):
+        # If we're not configured to rotate then abort.
+        if(not self.rotate_log_file):
+            return
+        
+        # So we have these files to possibly account for while we process the 
+        # rotation:- 
+        # self.log_file, self.log_file.0, self.log_file.1.gz, self.log_file.2.gz
+        # self.log_file.3.gz....self.log_file.(ROTATE_COUNT-1).gz
+        #
+        # Algorithm:-
+        # * Delete last file.
+        # * Work backwards moving 5->6, 4->5, 3->4, etc... but stop at .0
+        # * For .0 move it to .1 then gzip it.
+        # * Move self.log_file to .0
+        # * Done.
+        
+        # If it exists, remove the oldest file.
+        if(os.path.isfile(self.log_file + ".%d.gz" % (ROTATE_COUNT - 1))):
+            os.unlink(self.log_file + ".%d.gz" % (ROTATE_COUNT - 1))    
+
+        # Copy through the other gzipped log files.
+        for i in range(ROTATE_COUNT - 1, 1, -1):
+            src_file = self.log_file + ".%d.gz" % (i - 1)
+            dst_file = self.log_file + ".%d.gz" % i
+            
+            # If the source file exists move/rename it.
+            if(os.path.isfile(src_file)):
+                shutil.move(src_file, dst_file)
+        
+        # Now we need to handle the .0 -> .1.gz case.
+        if(os.path.isfile(self.log_file + ".0")):
+            # Move .0 to .1
+            shutil.move(self.log_file + ".0", self.log_file + ".1")
+            
+            # gzip the file.
+            fh = open(self.log_file + ".1", "r")
+            zfh = gzip.GzipFile(self.log_file + ".1.gz", "w")
+            zfh.write(fh.read())
+            zfh.flush()
+            zfh.close()
+            fh.close()
+            
+            # If gzip worked remove the original .1 file.
+            if(os.path.isfile(self.log_file + ".1.gz")):
+                os.unlink(self.log_file + ".1")
+        
+        # Finally move the current logfile to .0
+        shutil.move(self.log_file, self.log_file + ".0") 
+
+
+def stderr(text):
+    sys.stderr.write("%s\n" % text)
+
+def usage():
+    stderr("Usage: %s [OPTIONS]\n" % sys.argv[0])
+    stderr("Valid Options:-")
+    stderr("  --logfile=LOGFILE\t\t\tSpecify the logfile to process,\n"+\
+           "\t\t\t\t\tdefault: /var/log/syslog\n")
+    
+    stderr("  --configfile=CONFIGFILE\t\tSpecify the bbackupd config file,\n "+\
+           "\t\t\t\t\tdefault: /etc/box/bbackupd.conf\n")
+    
+    stderr("  --email-to=user at example.com\t\tSpecify the e-mail address(es)\n"+\
+           "\t\t\t\t\tto send the report to, default is to\n"+\
+           "\t\t\t\t\tdisplay the report on the console.\n")
+    
+    stderr("  --email-from=user at example.com\t\tSpecify the e-mail address(es)"+\
+           "\n\t\t\t\t\tto set the From: address to,\n "+\
+           "\t\t\t\t\tdefault: report at boxbackup\n")
+    
+    stderr("  --stats\t\t\t\tIncludes the usage stats retrieved from \n"+\
+           "\t\t\t\t\t'bbackupquery usage' in the report.\n")
+    
+    stderr("  --verbose\t\t\t\tList every file that was backed up to\n"+\
+           "\t\t\t\t\tthe server, default is to just display\n"+\
+           "\t\t\t\t\tthe summary.\n")
+    
+    stderr("  --rotate\t\t\t\tRotates the log files like logrotate\n"+\
+           "\t\t\t\t\twould, see the comments for a use-case.\n")
+
+def main():
+    # The defaults
+    logfile = "/var/log/syslog"
+    configfile = "/etc/box/bbackupd.conf"
+    email_to = None
+    email_from = "report at boxbackup"
+    rotate = False
+    verbose = False 
+    stats = False
+
+    # Parse the options
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "srvhl:c:t:f:", 
+                        ["help", "logfile=", "configfile=","email-to=", 
+                         "email-from=","rotate","verbose","stats"])
+    except getopt.GetoptError:
+        usage()
+        return
+    
+    for opt, arg in opts:
+        if(opt in ("--logfile","-l")):
+            logfile = arg
+        elif(opt in ("--configfile", "-c")):
+            configfile = arg
+        elif(opt in ("--email-to", "-t")):
+            email_to = arg
+        elif(opt in ("--email-from", "-f")):
+            email_from = arg
+        elif(opt in ("--rotate", "-r")):
+            rotate = True
+        elif(opt in ("--verbose", "-v")):
+            verbose = True
+        elif(opt in ("--stats", "-s")):
+            stats = True
+        elif(opt in ("--help", "-h")):
+            usage()
+            return
+    
+    # Run the reporter
+    bbr = BoxBackupReporter(configfile, logfile, email_to, email_from, 
+                            rotate, verbose, stats)
+    try:
+        bbr.run()
+        bbr.deliver()
+    except BoxBackupReporter.BoxBackupReporterError, error_msg:
+        print error_msg
+
+if __name__ == "__main__":
+    main()


Property changes on: box/trunk/contrib/bbreporter/bbreporter.py
___________________________________________________________________
Name: svn:executable
   + *




More information about the Boxbackup-commit mailing list