[Box Backup-commit] COMMIT r1342 - in box/chris/general: bin/bbackupd lib/backupclient
boxbackup-dev at fluffy.co.uk
boxbackup-dev at fluffy.co.uk
Sun Mar 4 00:18:30 GMT 2007
Author: chris
Date: 2007-03-04 00:18:30 +0000 (Sun, 04 Mar 2007)
New Revision: 1342
Modified:
box/chris/general/bin/bbackupd/BackupClientContext.cpp
box/chris/general/bin/bbackupd/BackupClientContext.h
box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.cpp
box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.h
box/chris/general/bin/bbackupd/BackupDaemon.cpp
box/chris/general/bin/bbackupd/BackupDaemon.h
box/chris/general/lib/backupclient/BackupStoreFile.h
Log:
BIG patch:
Use Timers to manage diffs and keepalives. Remove signal handler mess
from BackupClientContext.
Use ProgressNotifier to report upload status of each file.
Implement ProgressNotifier in bbackupd, using logging framework.
Add LogAllFileAccess option for verbose file access logging for debugging.
Add support for ExtendedLogFile directive to write extended (protocol) log
to a file instead of to the console/syslog.
Move MakeFullPath from BackupClientDirectoryRecord.cpp to PathUtils.cpp.
Send keepalives while scanning a directory, in case it's a very big or
slow directory.
Enable keepalives and diff time limits by default, at 60 and 600 seconds
respectively.
Don't delete remote directories just because we can't stat the local one.
Catch failure to read attributes from a location directory and log it
rather than aborting the backup run.
Log errors trying to write state to the command socket (doesn't affect win32).
(from chris/merge)
Modified: box/chris/general/bin/bbackupd/BackupClientContext.cpp
===================================================================
--- box/chris/general/bin/bbackupd/BackupClientContext.cpp 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/bin/bbackupd/BackupClientContext.cpp 2007-03-04 00:18:30 UTC (rev 1342)
@@ -35,13 +35,21 @@
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool)
+// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string)
// Purpose: Constructor
// Created: 2003/10/08
//
// --------------------------------------------------------------------------
-BackupClientContext::BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname,
- int32_t AccountNumber, bool ExtendedLogging)
+BackupClientContext::BackupClientContext
+(
+ BackupDaemon &rDaemon,
+ TLSContext &rTLSContext,
+ const std::string &rHostname,
+ int32_t AccountNumber,
+ bool ExtendedLogging,
+ bool ExtendedLogToFile,
+ std::string ExtendedLogFile
+)
: mrDaemon(rDaemon),
mrTLSContext(rTLSContext),
mHostname(rHostname),
@@ -49,6 +57,9 @@
mpSocket(0),
mpConnection(0),
mExtendedLogging(ExtendedLogging),
+ mExtendedLogToFile(ExtendedLogToFile),
+ mExtendedLogFile(ExtendedLogFile),
+ mpExtendedLogFileHandle(NULL),
mClientStoreMarker(ClientStoreMarker_NotKnown),
mpDeleteList(0),
mpCurrentIDMap(0),
@@ -56,8 +67,8 @@
mStorageLimitExceeded(false),
mpExcludeFiles(0),
mpExcludeDirs(0),
- mbIsManaged(false),
- mTimeMgmtEpoch(0)
+ mKeepAliveTimer(0),
+ mbIsManaged(false)
{
}
@@ -126,6 +137,24 @@
// Set logging option
mpConnection->SetLogToSysLog(mExtendedLogging);
+ if (mExtendedLogToFile)
+ {
+ ASSERT(mpExtendedLogFileHandle == NULL);
+
+ mpExtendedLogFileHandle = fopen(
+ mExtendedLogFile.c_str(), "a+");
+
+ if (!mpExtendedLogFileHandle)
+ {
+ ::syslog(LOG_ERR, "Failed to open extended "
+ "log file: %s", strerror(errno));
+ }
+ else
+ {
+ mpConnection->SetLogToFile(mpExtendedLogFileHandle);
+ }
+ }
+
// Handshake
mpConnection->Handshake();
@@ -256,6 +285,12 @@
delete mpDeleteList;
mpDeleteList = 0;
}
+
+ if (mpExtendedLogFileHandle != NULL)
+ {
+ fclose(mpExtendedLogFileHandle);
+ mpExtendedLogFileHandle = NULL;
+ }
}
@@ -303,8 +338,8 @@
// --------------------------------------------------------------------------
//
// Function
-// Name:
-// Purpose:
+// Name: BackupClientContext::PerformDeletions()
+// Purpose: Perform any pending file deletions.
// Created: 10/11/03
//
// --------------------------------------------------------------------------
@@ -461,40 +496,22 @@
return true;
}
-
-// maximum time to spend diffing
-static int sMaximumDiffTime = 600;
-// maximum time of SSL inactivity (keep-alive interval)
-static int sKeepAliveTime = 0;
-
void BackupClientContext::SetMaximumDiffingTime(int iSeconds)
{
- sMaximumDiffTime = iSeconds < 0 ? 0 : iSeconds;
- TRACE1("Set maximum diffing time to %d seconds\n", sMaximumDiffTime);
+ mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds;
+ TRACE1("Set maximum diffing time to %d seconds\n", mMaximumDiffingTime);
}
void BackupClientContext::SetKeepAliveTime(int iSeconds)
{
- sKeepAliveTime = iSeconds < 0 ? 0 : iSeconds;
- TRACE1("Set keep-alive time to %d seconds\n", sKeepAliveTime);
+ mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds;
+ TRACE1("Set keep-alive time to %d seconds\n", mKeepAliveTime);
+ mKeepAliveTimer = Timer(mKeepAliveTime);
}
// --------------------------------------------------------------------------
//
// Function
-// Name: static TimerSigHandler(int)
-// Purpose: Signal handler
-// Created: 19/3/04
-//
-// --------------------------------------------------------------------------
-static void TimerSigHandler(int iUnused)
-{
- BackupStoreFile::DiffTimerExpired();
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
// Name: BackupClientContext::ManageDiffProcess()
// Purpose: Initiates a file diff control timer
// Created: 04/19/2005
@@ -502,59 +519,8 @@
// --------------------------------------------------------------------------
void BackupClientContext::ManageDiffProcess()
{
- if (mbIsManaged || !mpConnection)
- return;
-
- ASSERT(mTimeMgmtEpoch == 0);
-
-#ifdef PLATFORM_CYGWIN
- ::signal(SIGALRM, TimerSigHandler);
-#elif defined WIN32
- // no support for SIGVTALRM
- SetTimerHandler(TimerSigHandler);
-#else
- ::signal(SIGVTALRM, TimerSigHandler);
-#endif // PLATFORM_CYGWIN
-
- struct itimerval timeout;
- memset(&timeout, 0, sizeof(timeout));
-
- //
- //
- //
- if (sMaximumDiffTime <= 0 && sKeepAliveTime <= 0)
- {
- TRACE0("Diff control not requested - letting things run wild\n");
- return;
- }
- else if (sMaximumDiffTime > 0 && sKeepAliveTime > 0)
- {
- timeout.it_value.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime;
- timeout.it_interval.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime;
- }
- else
- {
- timeout.it_value.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime;
- timeout.it_interval.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime;
- }
-
- // avoid race
- mTimeMgmtEpoch = time(NULL);
-
-#ifdef PLATFORM_CYGWIN
- if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0)
-#else
- if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
-#endif // PLATFORM_CYGWIN
- {
- mTimeMgmtEpoch = 0;
-
- TRACE0("WARNING: couldn't set file diff control timeout\n");
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-
+ ASSERT(!mbIsManaged);
mbIsManaged = true;
- TRACE0("Initiated timer for file diff control\n");
}
// --------------------------------------------------------------------------
@@ -567,33 +533,16 @@
// --------------------------------------------------------------------------
void BackupClientContext::UnManageDiffProcess()
{
- if (!mbIsManaged /* don't test for active connection, just do it */)
- return;
-
- struct itimerval timeout;
- memset(&timeout, 0, sizeof(timeout));
-
-#ifdef PLATFORM_CYGWIN
- if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0)
-#else
- if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
-#endif // PLATFORM_CYGWIN
- {
- TRACE0("WARNING: couldn't clear file diff control timeout\n");
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-
+ // ASSERT(mbIsManaged);
mbIsManaged = false;
- mTimeMgmtEpoch = 0;
-
- TRACE0("Suspended timer for file diff control\n");
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupClientContext::DoKeepAlive()
-// Purpose: Does something inconsequential over the SSL link to keep it up
+// Purpose: Check whether it's time to send a KeepAlive
+// message over the SSL link, and if so, send it.
// Created: 04/19/2005
//
// --------------------------------------------------------------------------
@@ -601,33 +550,26 @@
{
if (!mpConnection)
{
- ::syslog(LOG_ERR, "DoKeepAlive() called with no connection!");
return;
}
+
+ if (mKeepAliveTime == 0)
+ {
+ return;
+ }
+ if (!mKeepAliveTimer.HasExpired())
+ {
+ return;
+ }
+
+ TRACE0("KeepAliveTime reached, sending keep-alive message\n");
mpConnection->QueryGetIsAlive();
+
+ mKeepAliveTimer = Timer(mKeepAliveTime);
}
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientContext::GetTimeMgmtEpoch()
-// Purpose: Returns the unix time when the diff was started, or zero
-// if the diff process is unmanaged.
-// Created: 04/19/2005
-//
-// --------------------------------------------------------------------------
-time_t BackupClientContext::GetTimeMgmtEpoch()
-{
- return mTimeMgmtEpoch;
-}
-
int BackupClientContext::GetMaximumDiffingTime()
{
- return sMaximumDiffTime;
+ return mMaximumDiffingTime;
}
-
-int BackupClientContext::GetKeepaliveTime()
-{
- return sKeepAliveTime;
-}
Modified: box/chris/general/bin/bbackupd/BackupClientContext.h
===================================================================
--- box/chris/general/bin/bbackupd/BackupClientContext.h 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/bin/bbackupd/BackupClientContext.h 2007-03-04 00:18:30 UTC (rev 1342)
@@ -14,6 +14,7 @@
#include "BackupClientDeleteList.h"
#include "BackupStoreFile.h"
#include "ExcludeList.h"
+#include "Timer.h"
class TLSContext;
class BackupProtocolClient;
@@ -35,8 +36,16 @@
class BackupClientContext : public DiffTimer
{
public:
- BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname,
- int32_t AccountNumber, bool ExtendedLogging);
+ BackupClientContext
+ (
+ BackupDaemon &rDaemon,
+ TLSContext &rTLSContext,
+ const std::string &rHostname,
+ int32_t AccountNumber,
+ bool ExtendedLogging,
+ bool ExtendedLogToFile,
+ std::string ExtendedLogFile
+ );
virtual ~BackupClientContext();
private:
BackupClientContext(const BackupClientContext &);
@@ -143,7 +152,7 @@
// Created: 04/19/2005
//
// --------------------------------------------------------------------------
- static void SetMaximumDiffingTime(int iSeconds);
+ void SetMaximumDiffingTime(int iSeconds);
// --------------------------------------------------------------------------
//
@@ -153,7 +162,7 @@
// Created: 04/19/2005
//
// --------------------------------------------------------------------------
- static void SetKeepAliveTime(int iSeconds);
+ void SetKeepAliveTime(int iSeconds);
// --------------------------------------------------------------------------
//
@@ -175,19 +184,18 @@
// --------------------------------------------------------------------------
void UnManageDiffProcess();
- // --------------------------------------------------------------------------
+ // -------------------------------------------------------------------
//
// Function
// Name: BackupClientContext::DoKeepAlive()
- // Purpose: Does something inconsequential over the SSL link to
- // keep it up, implements DiffTimer interface
+ // Purpose: Check whether it's time to send a KeepAlive
+ // message over the SSL link, and if so, send it.
// Created: 04/19/2005
//
- // --------------------------------------------------------------------------
+ // -------------------------------------------------------------------
virtual void DoKeepAlive();
- virtual time_t GetTimeMgmtEpoch();
virtual int GetMaximumDiffingTime();
- virtual int GetKeepaliveTime();
+ virtual bool IsManaged() { return mbIsManaged; }
private:
BackupDaemon &mrDaemon;
@@ -197,6 +205,9 @@
SocketStreamTLS *mpSocket;
BackupProtocolClient *mpConnection;
bool mExtendedLogging;
+ bool mExtendedLogToFile;
+ std::string mExtendedLogFile;
+ FILE* mpExtendedLogFileHandle;
int64_t mClientStoreMarker;
BackupClientDeleteList *mpDeleteList;
const BackupClientInodeToIDMap *mpCurrentIDMap;
@@ -204,11 +215,10 @@
bool mStorageLimitExceeded;
ExcludeList *mpExcludeFiles;
ExcludeList *mpExcludeDirs;
-
+ Timer mKeepAliveTimer;
bool mbIsManaged;
- // unix time when diff was started
- time_t mTimeMgmtEpoch;
+ int mKeepAliveTime;
+ int mMaximumDiffingTime;
};
-
#endif // BACKUPCLIENTCONTEXT__H
Modified: box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.cpp
===================================================================
--- box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.cpp 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.cpp 2007-03-04 00:18:30 UTC (rev 1342)
@@ -29,6 +29,7 @@
#include "BackupDaemon.h"
#include "BackupStoreException.h"
#include "Archive.h"
+#include "PathUtils.h"
#include "MemLeakFindOn.h"
@@ -97,33 +98,6 @@
// --------------------------------------------------------------------------
//
// Function
-// Name: MakeFullPath(const std::string& rDir, const std::string& rFile)
-// Purpose: Combine directory and file name
-// Created: 2006/08/10
-//
-// --------------------------------------------------------------------------
-static std::string MakeFullPath(const std::string& rDir,
- const std::string& rFile)
-{
- std::string result;
-
- if (rDir.size() > 0 &&
- rDir[rDir.size()-1] == DIRECTORY_SEPARATOR_ASCHAR)
- {
- result = rDir + rFile;
- }
- else
- {
- result = rDir + DIRECTORY_SEPARATOR + rFile;
- }
-
- return result;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
// Name: BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool)
// Purpose: Syncronise, recusively, a local directory with the server.
// Created: 2003/10/08
@@ -164,8 +138,8 @@
{
// The directory has probably been deleted, so just ignore this error.
// In a future scan, this deletion will be noticed, deleted from server, and this object deleted.
- TRACE1("Stat failed for '%s' (directory)\n",
- rLocalPath.c_str());
+ rParams.GetProgressNotifier().NotifyDirStatFailed(
+ this, rLocalPath, strerror(errno));
return;
}
// Store inode number in map so directories are tracked in case they're renamed
@@ -193,15 +167,48 @@
std::vector<std::string> dirs;
std::vector<std::string> files;
bool downloadDirectoryRecordBecauseOfFutureFiles = false;
+
+ struct stat dir_st;
+ if(::lstat(rLocalPath.c_str(), &dir_st) != 0)
+ {
+ // Report the error (logs and
+ // eventual email to administrator)
+ rParams.GetProgressNotifier().NotifyFileStatFailed(this,
+ rLocalPath, strerror(errno));
+
+ // FIXME move to NotifyFileStatFailed()
+ SetErrorWhenReadingFilesystemObject(rParams,
+ rLocalPath.c_str());
+
+ // This shouldn't happen, so we'd better not continue
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
// BLOCK
{
// read the contents...
DIR *dirHandle = 0;
try
{
+ rParams.GetProgressNotifier().NotifyScanDirectory(
+ this, rLocalPath);
+
dirHandle = ::opendir(rLocalPath.c_str());
if(dirHandle == 0)
{
+ // Report the error (logs and
+ // eventual email to administrator)
+ if (errno == EACCES)
+ {
+ rParams.GetProgressNotifier().NotifyDirListFailed(
+ this, rLocalPath, "Access denied");
+ }
+ else
+ {
+ rParams.GetProgressNotifier().NotifyDirListFailed(this,
+ rLocalPath, strerror(errno));
+ }
+
// Report the error (logs and eventual email to administrator)
SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str());
// Ignore this directory for now.
@@ -223,6 +230,8 @@
std::string filename;
while((en = ::readdir(dirHandle)) != 0)
{
+ rParams.mrContext.DoKeepAlive();
+
// Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str());
// on Linux, as a stat is performed to get all this info
@@ -252,6 +261,10 @@
{
// Report the error (logs and
// eventual email to administrator)
+ rParams.GetProgressNotifier().NotifyFileStatFailed(this,
+ filename, strerror(errno));
+
+ // FIXME move to NotifyFileStatFailed()
SetErrorWhenReadingFilesystemObject(
rParams, filename.c_str());
@@ -259,6 +272,14 @@
continue;
}
+ if(st.st_dev != dir_st.st_dev)
+ {
+ rParams.GetProgressNotifier()
+ .NotifyMountPointSkipped(this,
+ filename);
+ continue;
+ }
+
int type = st.st_mode & S_IFMT;
#endif
@@ -269,6 +290,11 @@
// Exclude it?
if(rParams.mrContext.ExcludeFile(filename))
{
+ rParams.GetProgressNotifier()
+ .NotifyFileExcluded(
+ this,
+ filename);
+
// Next item!
continue;
}
@@ -283,6 +309,11 @@
// Exclude it?
if(rParams.mrContext.ExcludeDir(filename))
{
+ rParams.GetProgressNotifier()
+ .NotifyDirExcluded(
+ this,
+ filename);
+
// Next item!
continue;
}
@@ -292,13 +323,22 @@
}
else
{
- #ifdef WIN32
- ::syslog(LOG_ERR, "Unknown file type: "
- "%d (%s)", type,
- filename.c_str());
- #endif
- SetErrorWhenReadingFilesystemObject(
- rParams, filename.c_str());
+ if(rParams.mrContext.ExcludeFile(filename))
+ {
+ rParams.GetProgressNotifier()
+ .NotifyFileExcluded(
+ this,
+ filename);
+ }
+ else
+ {
+ rParams.GetProgressNotifier()
+ .NotifyUnsupportedFileType(
+ this, filename);
+ SetErrorWhenReadingFilesystemObject(
+ rParams, filename.c_str());
+ }
+
continue;
}
@@ -310,6 +350,11 @@
// but now we need the information.
if(::lstat(filename.c_str(), &st) != 0)
{
+ rParams.GetProgressNotifier()
+ .NotifyFileStatFailed(this,
+ filename,
+ strerror(errno));
+
// Report the error (logs and
// eventual email to administrator)
SetErrorWhenReadingFilesystemObject(
@@ -318,6 +363,14 @@
// Ignore this entry for now.
continue;
}
+
+ if(st.st_dev != dir_st.st_dev)
+ {
+ rParams.GetProgressNotifier()
+ .NotifyMountPointSkipped(this,
+ filename);
+ continue;
+ }
#endif
checksum_info.mModificationTime = FileModificationTime(st);
@@ -335,8 +388,8 @@
// Log that this has happened
if(!rParams.mHaveLoggedWarningAboutFutureFileTimes)
{
- ::syslog(LOG_ERR, "Some files have modification times excessively in the future. Check clock syncronisation.\n");
- ::syslog(LOG_ERR, "Example file (only one shown) : %s\n", filename.c_str());
+ rParams.GetProgressNotifier().NotifyFileModifiedInFuture(
+ this, filename);
rParams.mHaveLoggedWarningAboutFutureFileTimes = true;
}
}
@@ -574,6 +627,9 @@
for(std::vector<std::string>::const_iterator f = rFiles.begin();
f != rFiles.end(); ++f)
{
+ // Send keep-alive message if needed
+ rParams.mrContext.DoKeepAlive();
+
// Filename of this file
std::string filename(MakeFullPath(rLocalPath, *f));
@@ -589,7 +645,16 @@
struct stat st;
if(::lstat(filename.c_str(), &st) != 0)
{
- THROW_EXCEPTION(CommonException, OSFileError)
+ rParams.GetProgressNotifier().NotifyFileStatFailed(this,
+ filename, strerror(errno));
+
+ // Report the error (logs and
+ // eventual email to administrator)
+ SetErrorWhenReadingFilesystemObject(rParams,
+ filename.c_str());
+
+ // Ignore this entry for now.
+ continue;
}
// Extract required data
@@ -801,6 +866,9 @@
{
// Connection errors should just be passed on to the main handler, retries
// would probably just cause more problems.
+ rParams.GetProgressNotifier()
+ .NotifyFileUploadException(
+ this, filename, e);
throw;
}
catch(BoxException &e)
@@ -809,8 +877,9 @@
allUpdatedSuccessfully = false;
// Log it.
SetErrorWhenReadingFilesystemObject(rParams, filename.c_str());
- // Log error.
- ::syslog(LOG_ERR, "Error code when uploading was (%d/%d), %s", e.GetType(), e.GetSubType(), e.what());
+ rParams.GetProgressNotifier()
+ .NotifyFileUploadException(
+ this, filename, e);
}
// Update structures if the file was uploaded successfully.
@@ -823,6 +892,11 @@
}
}
}
+ else
+ {
+ rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this,
+ filename);
+ }
}
else if(en != 0 && en->GetAttributesHash() != attributesHash)
{
@@ -904,6 +978,9 @@
}
}
}
+
+ rParams.GetProgressNotifier().NotifyFileSynchronised(this,
+ filename, fileSize);
}
// Erase contents of files to save space when recursing
@@ -921,6 +998,9 @@
for(std::vector<std::string>::const_iterator d = rDirs.begin();
d != rDirs.end(); ++d)
{
+ // Send keep-alive message if needed
+ rParams.mrContext.DoKeepAlive();
+
// Get the local filename
std::string dirname(MakeFullPath(rLocalPath, *d));
@@ -1217,7 +1297,11 @@
if(diffFromID != 0)
{
- // Found an old version -- get the index
+ // Found an old version
+ rParams.GetProgressNotifier().NotifyFileUploadingPatch(this,
+ rFilename);
+
+ // Get the index
std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream());
//
@@ -1290,6 +1374,8 @@
throw;
}
+ rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize);
+
// Return the new object ID of this file
return objID;
}
@@ -1309,8 +1395,11 @@
// Zero hash, so it gets synced properly next time round.
::memset(mStateChecksum, 0, sizeof(mStateChecksum));
- // Log the error
- ::syslog(LOG_ERR, "Backup object failed, error when reading %s", Filename);
+ // Log the error - already done by caller
+ /*
+ rParams.GetProgressNotifier().NotifyFileReadFailed(this,
+ Filename, strerror(errno));
+ */
// Mark that an error occured in the parameters object
rParams.mReadErrorsOnFilesystemObjects = true;
@@ -1326,8 +1415,10 @@
// Created: 8/3/04
//
// --------------------------------------------------------------------------
-BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext)
- : mSyncPeriodStart(0),
+BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon,
+ ProgressNotifier &rProgressNotifier, BackupClientContext &rContext)
+ : mrProgressNotifier(rProgressNotifier),
+ mSyncPeriodStart(0),
mSyncPeriodEnd(0),
mMaxUploadWait(0),
mMaxFileTimeInFuture(99999999999999999LL),
Modified: box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.h
===================================================================
--- box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.h 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/bin/bbackupd/BackupClientDirectoryRecord.h 2007-03-04 00:18:30 UTC (rev 1342)
@@ -25,6 +25,78 @@
// --------------------------------------------------------------------------
//
// Class
+// Name: ProgressNotifier
+// Purpose: Provides methods for the backup library to inform the user
+// interface about its progress with the backup
+// Created: 2005/11/20
+//
+// --------------------------------------------------------------------------
+class BackupClientDirectoryRecord;
+
+class ProgressNotifier
+{
+ public:
+ virtual ~ProgressNotifier() { }
+ virtual void NotifyScanDirectory(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyDirStatFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg) = 0;
+ virtual void NotifyFileStatFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg) = 0;
+ virtual void NotifyDirListFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg) = 0;
+ virtual void NotifyMountPointSkipped(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyFileExcluded(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyDirExcluded(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyUnsupportedFileType(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyFileReadFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg) = 0;
+ virtual void NotifyFileModifiedInFuture(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyFileSkippedServerFull(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyFileUploadException(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const BoxException& rException) = 0;
+ virtual void NotifyFileUploading(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyFileUploadingPatch(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath) = 0;
+ virtual void NotifyFileUploaded(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ int64_t FileSize) = 0;
+ virtual void NotifyFileSynchronised(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ int64_t FileSize) = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
// Name: BackupClientDirectoryRecord
// Purpose: Implementation of record about directory for backup client
// Created: 2003/10/08
@@ -59,14 +131,18 @@
class SyncParams
{
public:
- SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext);
+ SyncParams(
+ BackupDaemon &rDaemon,
+ ProgressNotifier &rProgressNotifier,
+ BackupClientContext &rContext);
~SyncParams();
private:
// No copying
SyncParams(const SyncParams&);
SyncParams &operator=(const SyncParams&);
+ ProgressNotifier &mrProgressNotifier;
+
public:
-
// Data members are public, as accessors are not justified here
box_time_t mSyncPeriodStart;
box_time_t mSyncPeriodEnd;
@@ -81,6 +157,11 @@
// Member variables modified by syncing process
box_time_t mUploadAfterThisTimeInTheFuture;
bool mHaveLoggedWarningAboutFutureFileTimes;
+
+ ProgressNotifier& GetProgressNotifier() const
+ {
+ return mrProgressNotifier;
+ }
};
void SyncDirectory(SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath,
Modified: box/chris/general/bin/bbackupd/BackupDaemon.cpp
===================================================================
--- box/chris/general/bin/bbackupd/BackupDaemon.cpp 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/bin/bbackupd/BackupDaemon.cpp 2007-03-04 00:18:30 UTC (rev 1342)
@@ -72,6 +72,7 @@
#include "IOStreamGetLine.h"
#include "Conversion.h"
#include "Archive.h"
+#include "Timer.h"
#include "Logging.h"
#include "MemLeakFindOn.h"
@@ -112,7 +113,8 @@
BackupDaemon::BackupDaemon()
: mState(BackupDaemon::State_Initialising),
mpCommandSocketInfo(0),
- mDeleteUnusedRootDirEntriesAfter(0)
+ mDeleteUnusedRootDirEntriesAfter(0),
+ mLogAllFileAccess(false)
{
// Only ever one instance of a daemon
SSLLib::Initialise();
@@ -355,6 +357,9 @@
HANDLE handles[2];
handles[0] = mhMessageToSendEvent;
handles[1] = rSocket.GetReadableEvent();
+
+ BOX_TRACE("Received command '" << command
+ << "' over command socket");
DWORD result = WaitForMultipleObjects(
sizeof(handles)/sizeof(*handles),
@@ -503,21 +508,19 @@
// --------------------------------------------------------------------------
void BackupDaemon::Run()
{
+ // initialise global timer mechanism
+ Timers::Init();
+
#ifdef WIN32
- // init our own timer for file diff timeouts
- InitTimer();
-
try
{
Run2();
}
catch(...)
{
- FiniTimer();
+ Timers::Cleanup();
throw;
}
-
- FiniTimer();
#else // ! WIN32
// Ignore SIGPIPE (so that if a command connection is broken, the daemon doesn't terminate)
::signal(SIGPIPE, SIG_IGN);
@@ -560,6 +563,8 @@
mpCommandSocketInfo = 0;
}
+ Timers::Cleanup();
+
throw;
}
@@ -570,6 +575,8 @@
mpCommandSocketInfo = 0;
}
#endif
+
+ Timers::Cleanup();
}
// --------------------------------------------------------------------------
@@ -593,18 +600,20 @@
// Set up the keys for various things
BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
+ // Setup various timings
+ int maximumDiffingTime = 600;
+ int keepAliveTime = 60;
+
// max diffing time, keep-alive time
if(conf.KeyExists("MaximumDiffingTime"))
{
- BackupClientContext::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime"));
+ maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime");
}
if(conf.KeyExists("KeepAliveTime"))
{
- BackupClientContext::SetKeepAliveTime(conf.GetKeyValueInt("KeepAliveTime"));
+ keepAliveTime = conf.GetKeyValueInt("KeepAliveTime");
}
- // Setup various timings
-
// How often to connect to the store (approximate)
box_time_t updateStoreInterval = SecondsToBoxTime(conf.GetKeyValueInt("UpdateStoreInterval"));
@@ -767,18 +776,44 @@
SetState(State_Connected);
BOX_NOTICE("Beginning scan of local files");
- // Then create a client context object (don't just connect, as this may be unnecessary)
- BackupClientContext clientContext(*this, tlsContext, conf.GetKeyValue("StoreHostname"),
- conf.GetKeyValueInt("AccountNumber"), conf.GetKeyValueBool("ExtendedLogging"));
+ std::string extendedLogFile;
+ if (conf.KeyExists("ExtendedLogFile"))
+ {
+ extendedLogFile = conf.GetKeyValue(
+ "ExtendedLogFile");
+ }
+
+ if (conf.KeyExists("LogAllFileAccess"))
+ {
+ mLogAllFileAccess =
+ conf.GetKeyValueBool(
+ "LogAllFileAccess");
+ }
+
+ // Then create a client context object (don't
+ // just connect, as this may be unnecessary)
+ BackupClientContext clientContext
+ (
+ *this,
+ tlsContext,
+ conf.GetKeyValue("StoreHostname"),
+ conf.GetKeyValueInt("AccountNumber"),
+ conf.GetKeyValueBool("ExtendedLogging"),
+ conf.KeyExists("ExtendedLogFile"),
+ extendedLogFile
+ );
// Set up the sync parameters
- BackupClientDirectoryRecord::SyncParams params(*this, clientContext);
+ BackupClientDirectoryRecord::SyncParams params(*this, *this, clientContext);
params.mSyncPeriodStart = syncPeriodStart;
params.mSyncPeriodEnd = syncPeriodEndExtended; // use potentially extended end time
params.mMaxUploadWait = maxUploadWait;
params.mFileTrackingSizeThreshold = conf.GetKeyValueInt("FileTrackingSizeThreshold");
params.mDiffingUploadSizeThreshold = conf.GetKeyValueInt("DiffingUploadSizeThreshold");
params.mMaxFileTimeInFuture = SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture"));
+
+ clientContext.SetMaximumDiffingTime(maximumDiffingTime);
+ clientContext.SetKeepAliveTime(keepAliveTime);
// Set store marker
clientContext.SetClientStoreMarker(clientStoreMarker);
@@ -1529,7 +1564,23 @@
// Read the exclude lists from the Configuration
ploc->mpExcludeFiles = BackupClientMakeExcludeList_Files(i->second);
ploc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(i->second);
-
+ // Does this exist on the server?
+ // Remove from dir object early, so that if we fail
+ // to stat the local directory, we still don't
+ // consider to remote one for deletion.
+ BackupStoreDirectory::Iterator iter(dir);
+ BackupStoreFilenameClear dirname(ploc->mName); // generate the filename
+ BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname);
+ int64_t oid = 0;
+ if(en != 0)
+ {
+ oid = en->GetObjectID();
+
+ // Delete the entry from the directory, so we get a list of
+ // unused root directories at the end of this.
+ dir.DeleteEntry(oid);
+ }
+
// Do a fsstat on the pathname to find out which mount it's on
{
@@ -1607,26 +1658,26 @@
}
// Does this exist on the server?
- BackupStoreDirectory::Iterator iter(dir);
- BackupStoreFilenameClear dirname(ploc->mName); // generate the filename
- BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname);
- int64_t oid = 0;
- if(en != 0)
+ if(en == 0)
{
- oid = en->GetObjectID();
-
- // Delete the entry from the directory, so we get a list of
- // unused root directories at the end of this.
- dir.DeleteEntry(oid);
- }
- else
- {
// Doesn't exist, so it has to be created on the server. Let's go!
// First, get the directory's attributes and modification time
box_time_t attrModTime = 0;
BackupClientFileAttributes attr;
- attr.ReadAttributes(ploc->mPath.c_str(), true /* directories have zero mod times */,
- 0 /* not interested in mod time */, &attrModTime /* get the attribute modification time */);
+ try
+ {
+ attr.ReadAttributes(ploc->mPath.c_str(),
+ true /* directories have zero mod times */,
+ 0 /* not interested in mod time */,
+ &attrModTime /* get the attribute modification time */);
+ }
+ catch (BoxException &e)
+ {
+ BOX_ERROR("Failed to get attributes "
+ "for path '" << ploc->mPath
+ << "', skipping.");
+ continue;
+ }
// Execute create directory command
MemBlockStream attrStream(attr);
@@ -1650,6 +1701,9 @@
{
delete ploc;
ploc = 0;
+ BOX_ERROR("Failed to setup location '"
+ << ploc->mName << "' path '"
+ << ploc->mPath << "'");
throw;
}
}
@@ -1988,11 +2042,18 @@
// Something connected to the command socket, tell it about the new state
try
{
- mpCommandSocketInfo->mpConnectedSocket->Write(message.c_str(),
- message.length());
+ mpCommandSocketInfo->mpConnectedSocket->Write(newState, newStateSize);
}
+ catch(std::exception &e)
+ {
+ BOX_ERROR("Internal error while writing state "
+ "to command socket: " << e.what());
+ CloseCommandConnection();
+ }
catch(...)
{
+ BOX_ERROR("Internal error while writing state "
+ "to command socket: unknown error");
CloseCommandConnection();
}
#endif
Modified: box/chris/general/bin/bbackupd/BackupDaemon.h
===================================================================
--- box/chris/general/bin/bbackupd/BackupDaemon.h 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/bin/bbackupd/BackupDaemon.h 2007-03-04 00:18:30 UTC (rev 1342)
@@ -16,9 +16,12 @@
#include "BoxTime.h"
#include "Daemon.h"
+#include "BackupClientDirectoryRecord.h"
#include "Socket.h"
#include "SocketListen.h"
#include "SocketStream.h"
+#include "Logging.h"
+
#ifdef WIN32
#include "WinNamedPipeStream.h"
#endif
@@ -39,7 +42,7 @@
// Created: 2003/10/08
//
// --------------------------------------------------------------------------
-class BackupDaemon : public Daemon
+class BackupDaemon : public Daemon, ProgressNotifier
{
public:
BackupDaemon();
@@ -180,6 +183,163 @@
box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them
std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries;
+public:
+ bool StopRun() { return this->Daemon::StopRun(); }
+
+private:
+ bool mLogAllFileAccess;
+
+ /* ProgressNotifier implementation */
+public:
+ virtual void NotifyScanDirectory(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Scanning directory: " << rLocalPath);
+ }
+ }
+ virtual void NotifyDirStatFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg)
+ {
+ BOX_WARNING("Failed to access directory: " << rLocalPath
+ << ": " << rErrorMsg);
+ }
+ virtual void NotifyFileStatFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg)
+ {
+ BOX_WARNING("Failed to access file: " << rLocalPath
+ << ": " << rErrorMsg);
+ }
+ virtual void NotifyDirListFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg)
+ {
+ BOX_WARNING("Failed to list directory: " << rLocalPath
+ << ": " << rErrorMsg);
+ }
+ virtual void NotifyMountPointSkipped(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ BOX_WARNING("Ignored directory: " << rLocalPath << ": "
+ #ifdef WIN32
+ "is an NTFS junction/reparse point; "
+ #else
+ "is a mount point; "
+ #endif
+ "create a new location if you want to back it up");
+ }
+ virtual void NotifyFileExcluded(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Skipping excluded file: " << rLocalPath);
+ }
+ }
+ virtual void NotifyDirExcluded(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Skipping excluded directory: " << rLocalPath);
+ }
+ }
+ virtual void NotifyUnsupportedFileType(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ BOX_WARNING("Ignoring file of unknown type: " << rLocalPath);
+ }
+ virtual void NotifyFileReadFailed(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const std::string& rErrorMsg)
+ {
+ BOX_WARNING("Error reading file: " << rLocalPath
+ << ": " << rErrorMsg);
+ }
+ virtual void NotifyFileModifiedInFuture(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ BOX_WARNING("Some files have modification times excessively "
+ "in the future. Check clock synchronisation. "
+ "Example file (only one shown): " << rLocalPath);
+ }
+ virtual void NotifyFileSkippedServerFull(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ BOX_WARNING("Skipped file: server is full: " << rLocalPath);
+ }
+ virtual void NotifyFileUploadException(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ const BoxException& rException)
+ {
+ if (rException.GetType() == CommonException::ExceptionType &&
+ rException.GetSubType() == CommonException::AccessDenied)
+ {
+ BOX_ERROR("Failed to upload file: " << rLocalPath
+ << ": Access denied");
+ }
+ else
+ {
+ BOX_ERROR("Failed to upload file: " << rLocalPath
+ << ": caught exception: " << rException.what()
+ << " (" << rException.GetType()
+ << "/" << rException.GetSubType() << ")");
+ }
+ }
+ virtual void NotifyFileUploading(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Uploading file: " << rLocalPath);
+ }
+ }
+ virtual void NotifyFileUploadingPatch(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Uploading patch to file: " << rLocalPath);
+ }
+ }
+ virtual void NotifyFileUploaded(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ int64_t FileSize)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Uploaded file: " << rLocalPath);
+ }
+ }
+ virtual void NotifyFileSynchronised(
+ const BackupClientDirectoryRecord* pDirRecord,
+ const std::string& rLocalPath,
+ int64_t FileSize)
+ {
+ if (mLogAllFileAccess)
+ {
+ BOX_INFO("Synchronised file: " << rLocalPath);
+ }
+ }
+
#ifdef WIN32
public:
void RunHelperThread(void);
Modified: box/chris/general/lib/backupclient/BackupStoreFile.h
===================================================================
--- box/chris/general/lib/backupclient/BackupStoreFile.h 2007-03-03 23:57:23 UTC (rev 1341)
+++ box/chris/general/lib/backupclient/BackupStoreFile.h 2007-03-04 00:18:30 UTC (rev 1342)
@@ -50,17 +50,16 @@
DiffTimer();
virtual ~DiffTimer();
public:
- virtual void DoKeepAlive() = 0;
- virtual time_t GetTimeMgmtEpoch() = 0;
- virtual int GetMaximumDiffingTime() = 0;
- virtual int GetKeepaliveTime() = 0;
+ virtual void DoKeepAlive() = 0;
+ virtual int GetMaximumDiffingTime() = 0;
+ virtual bool IsManaged() = 0;
};
// --------------------------------------------------------------------------
//
// Class
// Name: BackupStoreFile
-// Purpose: Class to hold together utils for maniplating files.
+// Purpose: Class to hold together utils for manipulating files.
// Created: 2003/08/28
//
// --------------------------------------------------------------------------
More information about the Boxbackup-commit
mailing list