[Box Backup-dev] COMMIT r341 - in box/trunk: . bin/bbackupd bin/bbstored lib/backupclient test/backupdiff test/backupstore test/backupstorepatch
boxbackup-dev at fluffy.co.uk
boxbackup-dev at fluffy.co.uk
Sat Jan 28 00:13:51 GMT 2006
Author: chris
Date: 2006-01-28 00:13:44 +0000 (Sat, 28 Jan 2006)
New Revision: 341
Modified:
box/trunk/bin/bbackupd/BackupClientContext.cpp
box/trunk/bin/bbackupd/BackupClientContext.h
box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
box/trunk/bin/bbackupd/BackupDaemon.cpp
box/trunk/bin/bbstored/BackupCommands.cpp
box/trunk/bin/bbstored/backupprotocol.txt
box/trunk/configure.ac
box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp
box/trunk/lib/backupclient/BackupStoreFile.h
box/trunk/lib/backupclient/BackupStoreFileDiff.cpp
box/trunk/test/backupdiff/testbackupdiff.cpp
box/trunk/test/backupstore/testbackupstore.cpp
box/trunk/test/backupstorepatch/testbackupstorepatch.cpp
Log:
* configure.ac
* bin/bbstored/backupprotocol.txt
* bin/bbstored/BackupCommands.cpp
* bin/bbackupd/BackupClientContext.cpp
* bin/bbackupd/BackupClientContext.h
* bin/bbackupd/BackupClientDirectoryRecord.cpp
* bin/bbackupd/BackupDaemon.cpp
* lib/backupclient/BackupStoreFileDiff.cpp
* lib/backupclient/BackupDaemonConfigVerify.cpp
* lib/backupclient/BackupStoreFile.h
* test/backupstore/testbackupstore.cpp
* test/backupstorepatch/testbackupstorepatch.cpp
- Applied changes from chris/diff-timeout-and-ssl-keepalive
* test/backupdiff/testbackupdiff.cpp
- Fixed test to match new prototype for EncodeFileDiff
Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientContext.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/bin/bbackupd/BackupClientContext.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -9,9 +9,15 @@
#include "Box.h"
-#ifndef WIN32
-#include <syslog.h>
+#ifdef HAVE_SYSLOG_H
+ #include <syslog.h>
#endif
+#ifdef HAVE_SIGNAL_H
+ #include <signal.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+ #include <sys/time.h>
+#endif
#include "BoxPortsAndFiles.h"
#include "BoxTime.h"
@@ -22,6 +28,7 @@
#include "BackupStoreException.h"
#include "BackupDaemon.h"
#include "autogen_BackupProtocolClient.h"
+#include "BackupStoreFile.h"
#include "MemLeakFindOn.h"
@@ -48,7 +55,11 @@
mpNewIDMap(0),
mStorageLimitExceeded(false),
mpExcludeFiles(0),
- mpExcludeDirs(0)
+ mpExcludeDirs(0),
+ mbIsManaged(false),
+ mTimeMgmtEpoch(0),
+ mMaximumDiffTime(600),
+ mKeepAliveTime(0)
{
}
@@ -453,3 +464,166 @@
}
+// maximum time to spend diffing
+static int sMaximumDiffTime = 600;
+// maximum time of SSL inactivity (keep-alive interval)
+static int sKeepAliveTime = 0;
+
+void BackupClientContext::SetMaximumDiffingTime(int iSeconds)
+{
+ sMaximumDiffTime = iSeconds < 0 ? 0 : iSeconds;
+ TRACE1("Set maximum diffing time to %d seconds\n", sMaximumDiffTime);
+}
+
+void BackupClientContext::SetKeepAliveTime(int iSeconds)
+{
+ sKeepAliveTime = iSeconds < 0 ? 0 : iSeconds;
+ TRACE1("Set keep-alive time to %d seconds\n", sKeepAliveTime);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static TimerSigHandler(int)
+// Purpose: Signal handler
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+static void TimerSigHandler(int iUnused)
+{
+ BackupStoreFile::DiffTimerExpired();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientContext::ManageDiffProcess()
+// Purpose: Initiates a file diff control timer
+// Created: 04/19/2005
+//
+// --------------------------------------------------------------------------
+void BackupClientContext::ManageDiffProcess()
+{
+ if (mbIsManaged || !mpConnection)
+ return;
+
+ ASSERT(mTimeMgmtEpoch == 0);
+
+#ifdef PLATFORM_CYGWIN
+ ::signal(SIGALRM, TimerSigHandler);
+#else
+ ::signal(SIGVTALRM, TimerSigHandler);
+#endif // PLATFORM_CYGWIN
+
+ struct itimerval timeout;
+ memset(&timeout, 0, sizeof(timeout));
+
+ //
+ //
+ //
+ if (sMaximumDiffTime <= 0 && sKeepAliveTime <= 0)
+ {
+ TRACE0("Diff control not requested - letting things run wild\n");
+ return;
+ }
+ else if (sMaximumDiffTime > 0 && sKeepAliveTime > 0)
+ {
+ timeout.it_value.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime;
+ timeout.it_interval.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime;
+ }
+ else
+ {
+ timeout.it_value.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime;
+ timeout.it_interval.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime;
+ }
+
+ // avoid race
+ mTimeMgmtEpoch = time(NULL);
+
+#ifdef PLATFORM_CYGWIN
+ if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0)
+#else
+ if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
+#endif // PLATFORM_CYGWIN
+ {
+ mTimeMgmtEpoch = 0;
+
+ TRACE0("WARNING: couldn't set file diff control timeout\n");
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ mbIsManaged = true;
+ TRACE0("Initiated timer for file diff control\n");
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientContext::UnManageDiffProcess()
+// Purpose: suspends file diff control timer
+// Created: 04/19/2005
+//
+// --------------------------------------------------------------------------
+void BackupClientContext::UnManageDiffProcess()
+{
+ if (!mbIsManaged /* don't test for active connection, just do it */)
+ return;
+
+ struct itimerval timeout;
+ memset(&timeout, 0, sizeof(timeout));
+
+#ifdef PLATFORM_CYGWIN
+ if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0)
+#else
+ if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
+#endif // PLATFORM_CYGWIN
+ {
+ TRACE0("WARNING: couldn't clear file diff control timeout\n");
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ mbIsManaged = false;
+ mTimeMgmtEpoch = 0;
+
+ TRACE0("Suspended timer for file diff control\n");
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientContext::DoKeepAlive()
+// Purpose: Does something inconsequential over the SSL link to keep it up
+// Created: 04/19/2005
+//
+// --------------------------------------------------------------------------
+void BackupClientContext::DoKeepAlive()
+{
+ if (!mpConnection)
+ return;
+
+ mpConnection->QueryGetIsAlive();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientContext::GetTimeMgmtEpoch()
+// Purpose: Returns the unix time when the diff was started, or zero
+// if the diff process is unmanaged.
+// Created: 04/19/2005
+//
+// --------------------------------------------------------------------------
+time_t BackupClientContext::GetTimeMgmtEpoch()
+{
+ return mTimeMgmtEpoch;
+}
+
+int BackupClientContext::GetMaximumDiffingTime()
+{
+ return mMaximumDiffTime;
+}
+
+int BackupClientContext::GetKeepaliveTime()
+{
+ return mKeepAliveTime;
+}
Modified: box/trunk/bin/bbackupd/BackupClientContext.h
===================================================================
--- box/trunk/bin/bbackupd/BackupClientContext.h 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/bin/bbackupd/BackupClientContext.h 2006-01-28 00:13:44 UTC (rev 341)
@@ -12,6 +12,7 @@
#include "BoxTime.h"
#include "BackupClientDeleteList.h"
+#include "BackupStoreFile.h"
#include "ExcludeList.h"
class TLSContext;
@@ -31,12 +32,12 @@
// Created: 2003/10/08
//
// --------------------------------------------------------------------------
-class BackupClientContext
+class BackupClientContext : public DiffTimer
{
public:
BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname,
int32_t AccountNumber, bool ExtendedLogging);
- ~BackupClientContext();
+ virtual ~BackupClientContext();
private:
BackupClientContext(const BackupClientContext &);
public:
@@ -134,6 +135,60 @@
bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0,
BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BackupClientContext::SetMaximumDiffingTime()
+ // Purpose: Sets the maximum time that will be spent diffing a file
+ // Created: 04/19/2005
+ //
+ // --------------------------------------------------------------------------
+ static void SetMaximumDiffingTime(int iSeconds);
+
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BackupClientContext::SetKeepAliveTime()
+ // Purpose: Sets the time interval for repetitive keep-alive operation
+ // Created: 04/19/2005
+ //
+ // --------------------------------------------------------------------------
+ static void SetKeepAliveTime(int iSeconds);
+
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BackupClientContext::ManageDiffProcess()
+ // Purpose: Initiates an SSL connection/session keep-alive process
+ // Created: 04/19/2005
+ //
+ // --------------------------------------------------------------------------
+ void ManageDiffProcess();
+
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BackupClientContext::UnManageDiffProcess()
+ // Purpose: Suspends an SSL connection/session keep-alive process
+ // Created: 04/19/2005
+ //
+ // --------------------------------------------------------------------------
+ void UnManageDiffProcess();
+
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BackupClientContext::DoKeepAlive()
+ // Purpose: Does something inconsequential over the SSL link to
+ // keep it up, implements DiffTimer interface
+ // Created: 04/19/2005
+ //
+ // --------------------------------------------------------------------------
+ virtual void DoKeepAlive();
+ virtual time_t GetTimeMgmtEpoch();
+ virtual int GetMaximumDiffingTime();
+ virtual int GetKeepaliveTime();
+
private:
BackupDaemon &mrDaemon;
TLSContext &mrTLSContext;
@@ -149,8 +204,16 @@
bool mStorageLimitExceeded;
ExcludeList *mpExcludeFiles;
ExcludeList *mpExcludeDirs;
+
+ bool mbIsManaged;
+ // unix time when diff was started
+ time_t mTimeMgmtEpoch;
+ // maximum time to spend diffing, in seconds
+ int mMaximumDiffTime;
+ // maximum time of SSL inactivity (keep-alive interval), in seconds
+ int mKeepAliveTime;
+
};
#endif // BACKUPCLIENTCONTEXT__H
-
Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -1093,14 +1093,28 @@
// Found an old version -- get the index
std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream());
+ //
// Diff the file
+ //
+
+ rParams.mrContext.ManageDiffProcess();
+
bool isCompletelyDifferent = false;
- std::auto_ptr<IOStream> patchStream(BackupStoreFile::EncodeFileDiff(rFilename.c_str(),
+ std::auto_ptr<IOStream> patchStream(
+ BackupStoreFile::EncodeFileDiff(
+ rFilename.c_str(),
mObjectID, /* containing directory */
rStoreFilename, diffFromID, *blockIndexStream,
- connection.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent));
+ connection.GetTimeout(),
+ &rParams.mrContext, // DiffTimer implementation
+ 0 /* not interested in the modification time */,
+ &isCompletelyDifferent));
+ rParams.mrContext.UnManageDiffProcess();
+
+ //
// Upload the patch to the store
+ //
std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime,
AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream));
@@ -1130,6 +1144,8 @@
}
catch(BoxException &e)
{
+ rParams.mrContext.UnManageDiffProcess();
+
if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply)
{
// Check and see what error the protocol has -- as it might be an error...
@@ -1210,6 +1226,3 @@
BackupClientDirectoryRecord::SyncParams::~SyncParams()
{
}
-
-
-
Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupDaemon.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/bin/bbackupd/BackupDaemon.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -435,11 +435,15 @@
// Set up the keys for various things
BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
- // Set maximum diffing time?
+ // max diffing time, keep-alive time
if(conf.KeyExists("MaximumDiffingTime"))
{
- BackupStoreFile::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime"));
+ BackupClientContext::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime"));
}
+ if(conf.KeyExists("KeepAliveTime"))
+ {
+ BackupClientContext::SetKeepAliveTime(conf.GetKeyValueInt("KeepAliveTime"));
+ }
// Setup various timings
@@ -885,7 +889,7 @@
{
#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
bool uidOK = true;
- ::syslog(LOG_ERR, "On this platform, no security check can be made on the credientials of peers connecting to the command socket. (bbackupctl)");
+ ::syslog(LOG_WARNING, "On this platform, no security check can be made on the credientials of peers connecting to the command socket. (bbackupctl)");
#else
// Security check -- does the process connecting to this socket have
// the same UID as this process?
Modified: box/trunk/bin/bbstored/BackupCommands.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupCommands.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/bin/bbstored/BackupCommands.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -859,3 +859,20 @@
));
}
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupContext &)
+// Purpose: Return the amount of disc space used
+// Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ //
+ // NOOP
+ //
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerIsAlive());
+}
Modified: box/trunk/bin/bbstored/backupprotocol.txt
===================================================================
--- box/trunk/bin/bbstored/backupprotocol.txt 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/bin/bbstored/backupprotocol.txt 2006-01-28 00:13:44 UTC (rev 341)
@@ -219,3 +219,10 @@
int64 BlocksSoftLimit
int64 BlocksHardLimit
int32 BlockSize
+
+GetIsAlive 42 Command(IsAlive)
+ # no data members
+
+IsAlive 43 Reply
+ # no data members
+
Modified: box/trunk/configure.ac
===================================================================
--- box/trunk/configure.ac 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/configure.ac 2006-01-28 00:13:44 UTC (rev 341)
@@ -75,7 +75,8 @@
AC_HEADER_STDC
AC_HEADER_SYS_WAIT
AC_CHECK_HEADERS([execinfo.h netinet/in.h regex.h sys/types.h sys/xattr.h])
-AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h])
+AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h syslog.h signal.h sys/time.h])
+AC_CHECK_HEADERS([time.h])
### Checks for typedefs, structures, and compiler characteristics.
Modified: box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -84,6 +84,7 @@
{"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
{"CommandSocket", 0, 0, 0}, // not compulsory to have this
+ {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional
{"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full
Modified: box/trunk/lib/backupclient/BackupStoreFile.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFile.h 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/lib/backupclient/BackupStoreFile.h 2006-01-28 00:13:44 UTC (rev 341)
@@ -38,6 +38,24 @@
// --------------------------------------------------------------------------
//
// Class
+// Name: DiffTimer
+// Purpose: Interface for classes that can keep track of diffing time,
+// and send SSL keepalive messages
+// Created: 2006/01/19
+//
+// --------------------------------------------------------------------------
+class DiffTimer
+{
+public:
+ virtual void DoKeepAlive() = 0;
+ virtual time_t GetTimeMgmtEpoch() = 0;
+ virtual int GetMaximumDiffingTime() = 0;
+ virtual int GetKeepaliveTime() = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
// Name: BackupStoreFile
// Purpose: Class to hold together utils for maniplating files.
// Created: 2003/08/28
@@ -95,9 +113,16 @@
// Main interface
static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0);
- static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID,
- const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
- int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0);
+ static std::auto_ptr<IOStream> EncodeFileDiff
+ (
+ const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename,
+ int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout,
+ DiffTimer *pDiffTimer,
+ int64_t *pModificationTime = 0,
+ bool *pIsCompletelyDifferent = 0
+ );
static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
@@ -144,8 +169,7 @@
free(a);
}
- // Limits
- static void SetMaximumDiffingTime(int Seconds);
+ static void DiffTimerExpired();
// Building blocks
class EncodingBuffer
@@ -192,4 +216,3 @@
#include "MemLeakFindOff.h"
#endif // BACKUPSTOREFILE__H
-
Modified: box/trunk/lib/backupclient/BackupStoreFileDiff.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileDiff.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/lib/backupclient/BackupStoreFileDiff.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -11,11 +11,11 @@
#include <new>
#include <map>
-#include <signal.h>
-#ifdef WIN32
-#include <time.h>
-#else
-#include <sys/time.h>
+
+#ifdef HAVE_TIME_H
+ #include <time.h>
+#elif HAVE_SYS_TIME_H
+ #include <sys/time.h>
#endif
#include "BackupStoreFile.h"
@@ -43,33 +43,37 @@
static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis);
static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
-static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SearchForMatchingBlocks(IOStream &rFile,
+ std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex,
+ int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES],
+ DiffTimer *pDiffTimer);
static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable);
static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber,
BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile);
-// Avoid running on too long with diffs
-static int sMaximumDiffTime = 600; // maximum time to spend diffing
-static bool sDiffTimedOut = false;
-static bool sSetTimerSignelHandler = false;
-static void TimerSignalHandler(int signal);
-static void StartDiffTimer();
+// sDiffTimerExpired flags when the diff timer has expired. When true, the
+// diff routine should check the wall clock as soon as possible, to determine
+// whether it's time for a keepalive to be sent, or whether the diff has been
+// running for too long and should be terminated.
+static bool sDiffTimerExpired = false;
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreFile::SetMaximumDiffingTime(int)
-// Purpose: Sets the maximum time to spend diffing, in seconds. Time is
-// process virutal time.
-// Created: 19/3/04
+// Name: BackupStoreFile::DiffTimerExpired()
+// Purpose: Notifies BackupStoreFile object that the diff operation
+// timer has expired, which may mean that a keepalive should
+// be sent, or the diff should be terminated. Called from an
+// external timer, so it should not do more than set a flag.
//
+// Created: 19/1/06
+//
// --------------------------------------------------------------------------
-void BackupStoreFile::SetMaximumDiffingTime(int Seconds)
+void BackupStoreFile::DiffTimerExpired()
{
- sMaximumDiffTime = Seconds;
- TRACE1("Set maximum diffing time to %d seconds\n", Seconds);
+ sDiffTimerExpired = true;
}
@@ -139,9 +143,12 @@
// Created: 12/1/04
//
// --------------------------------------------------------------------------
-std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID,
- const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
- int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent)
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff
+(
+ const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID,
+ IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer,
+ int64_t *pModificationTime, bool *pIsCompletelyDifferent)
{
// Is it a symlink?
{
@@ -180,9 +187,6 @@
// Pointer to recipe we're going to create
BackupStoreFileEncodeStream::Recipe *precipe = 0;
-
- // Start the timeout timer, so that the operation doesn't continue for ever
- StartDiffTimer();
try
{
@@ -204,7 +208,8 @@
// Get size of file
sizeOfInputFile = file.BytesLeftToRead();
// Find all those lovely matching blocks
- SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan);
+ SearchForMatchingBlocks(file, foundBlocks, pindex,
+ blocksInIndex, sizesToScan, pDiffTimer);
// Is it completely different?
completelyDifferent = (foundBlocks.size() == 0);
@@ -475,8 +480,20 @@
//
// --------------------------------------------------------------------------
static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
- BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+ BlocksAvailableEntry *pIndex, int64_t NumBlocks,
+ int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer)
{
+ time_t TimeMgmtEpoch = 0;
+ int MaximumDiffingTime = 0;
+ int KeepAliveTime = 0;
+
+ if (pDiffTimer)
+ {
+ TimeMgmtEpoch = pDiffTimer->GetTimeMgmtEpoch();
+ MaximumDiffingTime = pDiffTimer->GetMaximumDiffingTime();
+ KeepAliveTime = pDiffTimer->GetKeepaliveTime();
+ }
+
std::map<int64_t, int32_t> goodnessOfFit;
// Allocate the hash lookup table
@@ -643,9 +660,40 @@
// End this loop, so the final byte isn't used again
break;
}
+
+ bool DiffTimedOut = false;
- if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE) || sDiffTimedOut)
+ if(sDiffTimerExpired)
{
+ ASSERT(TimeMgmtEpoch > 0);
+ ASSERT(pDiffTimer != NULL);
+
+ time_t tTotalRunIntvl = time(NULL) - TimeMgmtEpoch;
+
+ if(MaximumDiffingTime > 0 &&
+ tTotalRunIntvl >= MaximumDiffingTime)
+ {
+ TRACE0("MaximumDiffingTime reached - "
+ "suspending file diff\n");
+ DiffTimedOut = true;
+ }
+ else if(KeepAliveTime > 0)
+ {
+ TRACE0("KeepAliveTime reached - "
+ "initiating keep-alive\n");
+ pDiffTimer->DoKeepAlive();
+ }
+
+ sDiffTimerExpired = false;
+ }
+
+ int64_t NumBlocksFound = static_cast<int64_t>(
+ rFoundBlocks.size());
+ int64_t MaxBlocksFound = NumBlocks *
+ BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE;
+
+ if(NumBlocksFound > MaxBlocksFound || DiffTimedOut)
+ {
abortSearch = true;
break;
}
@@ -1005,55 +1053,3 @@
}
#endif
}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: static TimerSignalHandler(int)
-// Purpose: Signal handler
-// Created: 19/3/04
-//
-// --------------------------------------------------------------------------
-void TimerSignalHandler(int signal)
-{
- sDiffTimedOut = true;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: static StartDiffTimer()
-// Purpose: Starts the diff timeout timer
-// Created: 19/3/04
-//
-// --------------------------------------------------------------------------
-void StartDiffTimer()
-{
- // Set timer signal handler
- if(!sSetTimerSignelHandler)
- {
- ::signal(SIGVTALRM, TimerSignalHandler);
- sSetTimerSignelHandler = true;
- }
-
- struct itimerval timeout;
- // Don't want this to repeat
- timeout.it_interval.tv_sec = 0;
- timeout.it_interval.tv_usec = 0;
- // Single timeout after the specified number of seconds
- timeout.it_value.tv_sec = sMaximumDiffTime;
- timeout.it_value.tv_usec = 0;
- // Set timer
- if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
- {
- TRACE0("WARNING: couldn't set diff timeout\n");
- }
-
- // Unset flag (last thing)
- sDiffTimedOut = false;
-}
-
-
-
Modified: box/trunk/test/backupdiff/testbackupdiff.cpp
===================================================================
--- box/trunk/test/backupdiff/testbackupdiff.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/test/backupdiff/testbackupdiff.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -169,9 +169,17 @@
{
BackupStoreFilenameClear f1name("filename");
FileStream out(to_diff, O_WRONLY | O_CREAT | O_EXCL);
- std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff(to_orig, 1 /* dir ID */, f1name,
- 1000 + from /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
- 0, &completelyDifferent));
+ std::auto_ptr<IOStream> encoded(
+ BackupStoreFile::EncodeFileDiff(
+ to_orig,
+ 1 /* dir ID */,
+ f1name,
+ 1000 + from /* object ID of the file diffing from */,
+ blockindex,
+ IOStream::TimeOutInfinite,
+ NULL, // DiffTimer interface
+ 0,
+ &completelyDifferent));
encoded->CopyStreamTo(out);
}
TEST_THAT(completelyDifferent == expect_completely_different);
@@ -443,9 +451,17 @@
BackupStoreFilenameClear f1name("filename");
FileStream out("testfiles/f2.symlink.diff", O_WRONLY | O_CREAT | O_EXCL);
- std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("testfiles/f2.symlink", 1 /* dir ID */, f1name,
- 1001 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
- 0, &completelyDifferent));
+ std::auto_ptr<IOStream> encoded(
+ BackupStoreFile::EncodeFileDiff(
+ "testfiles/f2.symlink",
+ 1 /* dir ID */,
+ f1name,
+ 1001 /* object ID of the file diffing from */,
+ blockindex,
+ IOStream::TimeOutInfinite,
+ NULL, // DiffTimer interface
+ 0,
+ &completelyDifferent));
encoded->CopyStreamTo(out);
}
TEST_THAT(completelyDifferent == true);
Modified: box/trunk/test/backupstore/testbackupstore.cpp
===================================================================
--- box/trunk/test/backupstore/testbackupstore.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/test/backupstore/testbackupstore.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -1072,9 +1072,16 @@
// Do the patching
bool isCompletelyDifferent = false;
int64_t modtime;
- std::auto_ptr<IOStream> patchstream(BackupStoreFile::EncodeFileDiff(TEST_FILE_FOR_PATCHING ".mod", BackupProtocolClientListDirectory::RootDirectory,
- uploads[UPLOAD_PATCH_EN].name, uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream,
- IOStream::TimeOutInfinite, &modtime, &isCompletelyDifferent));
+ std::auto_ptr<IOStream> patchstream(
+ BackupStoreFile::EncodeFileDiff(
+ TEST_FILE_FOR_PATCHING ".mod",
+ BackupProtocolClientListDirectory::RootDirectory,
+ uploads[UPLOAD_PATCH_EN].name,
+ uploads[UPLOAD_PATCH_EN].allocated_objid,
+ *blockIndexStream,
+ IOStream::TimeOutInfinite,
+ NULL, // pointer to DiffTimer impl
+ &modtime, &isCompletelyDifferent));
TEST_THAT(isCompletelyDifferent == false);
// Sent this to a file, so we can check the size, rather than uploading it directly
{
Modified: box/trunk/test/backupstorepatch/testbackupstorepatch.cpp
===================================================================
--- box/trunk/test/backupstorepatch/testbackupstorepatch.cpp 2006-01-27 22:31:17 UTC (rev 340)
+++ box/trunk/test/backupstorepatch/testbackupstorepatch.cpp 2006-01-28 00:13:44 UTC (rev 341)
@@ -376,10 +376,17 @@
char filename[64];
::sprintf(filename, "testfiles/%d.test", f);
bool isCompletelyDifferent = false;
- std::auto_ptr<IOStream> patchStream(BackupStoreFile::EncodeFileDiff(filename,
+ std::auto_ptr<IOStream> patchStream(
+ BackupStoreFile::EncodeFileDiff(
+ filename,
BackupProtocolClientListDirectory::RootDirectory, /* containing directory */
- storeFilename, diffFromID, *blockIndexStream,
- protocol.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent));
+ storeFilename,
+ diffFromID,
+ *blockIndexStream,
+ protocol.GetTimeout(),
+ NULL, // DiffTimer impl
+ 0 /* not interested in the modification time */,
+ &isCompletelyDifferent));
// Upload the patch to the store
std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
More information about the Boxbackup-dev
mailing list