From subversion at boxbackup.org Tue Nov 6 23:57:04 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Tue, 6 Nov 2012 23:57:04 GMT Subject: [Box Backup-commit] COMMIT r3128 - box/trunk/lib/common Message-ID: <201211062357.qA6Nv4uV058277@wm.boxbackup.org> Author: chris Date: 2012-11-06 23:57:03 +0000 (Tue, 06 Nov 2012) New Revision: 3128 Modified: box/trunk/lib/common/Logging.h box/trunk/lib/common/Timer.cpp Log: Convert FORMAT_MICROSECONDS into a global utility macro. Modified: box/trunk/lib/common/Logging.h =================================================================== --- box/trunk/lib/common/Logging.h 2012-10-22 20:56:04 UTC (rev 3127) +++ box/trunk/lib/common/Logging.h 2012-11-06 23:57:03 UTC (rev 3128) @@ -144,6 +144,10 @@ std::setw(6) << \ timespec.tv_usec +#define BOX_FORMAT_MICROSECONDS(t) \ + (int)((t) / 1000000) << "." << \ + (int)((t) % 1000000) << " seconds" + #undef ERROR namespace Log Modified: box/trunk/lib/common/Timer.cpp =================================================================== --- box/trunk/lib/common/Timer.cpp 2012-10-22 20:56:04 UTC (rev 3127) +++ box/trunk/lib/common/Timer.cpp 2012-11-06 23:57:03 UTC (rev 3128) @@ -170,10 +170,6 @@ } } -#define FORMAT_MICROSECONDS(t) \ - (int)(t / 1000000) << "." << \ - (int)(t % 1000000) << " seconds" - // -------------------------------------------------------------------------- // // Function @@ -242,7 +238,7 @@ */ BOX_TRACE(TIMER_ID_OF(**i) "has expired, " "triggering " << - FORMAT_MICROSECONDS(-timeToExpiry) << + BOX_FORMAT_MICROSECONDS(-timeToExpiry) << " late"); rTimer.OnExpire(); spTimers->erase(i); @@ -295,7 +291,7 @@ else { BOX_TRACE("timer: next event: " << nameOfNextEvent << - " expires in " << FORMAT_MICROSECONDS(timeToNextEvent)); + " expires in " << BOX_FORMAT_MICROSECONDS(timeToNextEvent)); } struct itimerval timeout; @@ -397,7 +393,7 @@ if (timeToExpiry <= 0) { BOX_WARNING(TIMER_ID << "fudging expiry from -" << - FORMAT_MICROSECONDS(-timeToExpiry)) + BOX_FORMAT_MICROSECONDS(-timeToExpiry)) timeToExpiry = 1; } From subversion at boxbackup.org Tue Nov 6 23:58:14 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Tue, 6 Nov 2012 23:58:14 GMT Subject: [Box Backup-commit] COMMIT r3129 - box/trunk/test/common Message-ID: <201211062358.qA6NwE2S058296@wm.boxbackup.org> Author: chris Date: 2012-11-06 23:58:14 +0000 (Tue, 06 Nov 2012) New Revision: 3129 Modified: box/trunk/test/common/testcommon.cpp Log: Timer arguments are in milliseconds, not seconds. Modified: box/trunk/test/common/testcommon.cpp =================================================================== --- box/trunk/test/common/testcommon.cpp 2012-11-06 23:57:03 UTC (rev 3128) +++ box/trunk/test/common/testcommon.cpp 2012-11-06 23:58:14 UTC (rev 3129) @@ -332,9 +332,9 @@ Timers::Init(); Timer t0(0, "t0"); // should never expire - Timer t1(1, "t1"); - Timer t2(2, "t2"); - Timer t3(3, "t3"); + Timer t1(1000, "t1"); + Timer t2(2000, "t2"); + Timer t3(3000, "t3"); TEST_THAT(!t0.HasExpired()); TEST_THAT(!t1.HasExpired()); @@ -353,8 +353,8 @@ TEST_THAT(t2.HasExpired()); TEST_THAT(!t3.HasExpired()); - t1 = Timer(1, "t1a"); - t2 = Timer(2, "t2a"); + t1 = Timer(1000, "t1a"); + t2 = Timer(2000, "t2a"); TEST_THAT(!t0.HasExpired()); TEST_THAT(!t1.HasExpired()); TEST_THAT(!t2.HasExpired()); From subversion at boxbackup.org Wed Nov 7 00:02:32 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 7 Nov 2012 00:02:32 GMT Subject: [Box Backup-commit] COMMIT r3130 - box/trunk/lib/common Message-ID: <201211070002.qA702W8s058344@wm.boxbackup.org> Author: chris Date: 2012-11-07 00:02:32 +0000 (Wed, 07 Nov 2012) New Revision: 3130 Modified: box/trunk/lib/common/BoxTime.cpp Log: Log the number of nanoseconds left to sleep, otherwise the log messages don't make sense. Modified: box/trunk/lib/common/BoxTime.cpp =================================================================== --- box/trunk/lib/common/BoxTime.cpp 2012-11-06 23:58:14 UTC (rev 3129) +++ box/trunk/lib/common/BoxTime.cpp 2012-11-07 00:02:32 UTC (rev 3130) @@ -127,7 +127,7 @@ // when nanosleep() returns later than expected. int32_t secs = (int32_t) ts.tv_sec; - int64_t remain_ns = (secs * 1000000000) + ts.tv_nsec; + int64_t remain_ns = ((int64_t)secs * 1000000000) + ts.tv_nsec; if (remain_ns < 0) { @@ -137,9 +137,8 @@ return; } - BOX_TRACE("nanosleep interrupted with " << - (remain_ns / 1000000000) << " secs remaining, " - "sleeping again"); + BOX_TRACE("nanosleep interrupted with " << remain_ns << + " nanosecs remaining, sleeping again"); } #endif } From subversion at boxbackup.org Wed Nov 7 00:08:14 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 7 Nov 2012 00:08:14 GMT Subject: [Box Backup-commit] COMMIT r3131 - box/trunk/lib/common Message-ID: <201211070008.qA708Ea9058381@wm.boxbackup.org> Author: chris Date: 2012-11-07 00:08:14 +0000 (Wed, 07 Nov 2012) New Revision: 3131 Modified: box/trunk/lib/common/Logging.h Log: BOX_FORMAT_MICROSECONDS doesn't make sense unless the digits to the right of the decimal point are padded properly. Modified: box/trunk/lib/common/Logging.h =================================================================== --- box/trunk/lib/common/Logging.h 2012-11-07 00:02:32 UTC (rev 3130) +++ box/trunk/lib/common/Logging.h 2012-11-07 00:08:14 UTC (rev 3131) @@ -146,6 +146,8 @@ #define BOX_FORMAT_MICROSECONDS(t) \ (int)((t) / 1000000) << "." << \ + std::setw(6) << \ + std::setfill('0') << \ (int)((t) % 1000000) << " seconds" #undef ERROR From subversion at boxbackup.org Wed Nov 7 00:24:37 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 7 Nov 2012 00:24:37 GMT Subject: [Box Backup-commit] COMMIT r3132 - box/trunk/lib/common Message-ID: <201211070024.qA70ObRj058476@wm.boxbackup.org> Author: chris Date: 2012-11-07 00:24:37 +0000 (Wed, 07 Nov 2012) New Revision: 3132 Modified: box/trunk/lib/common/Timer.cpp box/trunk/lib/common/Timer.h Log: Refactor and improve timer trace logging, fix subsecond formatting error. Modified: box/trunk/lib/common/Timer.cpp =================================================================== --- box/trunk/lib/common/Timer.cpp 2012-11-07 00:08:14 UTC (rev 3131) +++ box/trunk/lib/common/Timer.cpp 2012-11-07 00:24:37 UTC (rev 3132) @@ -30,7 +30,7 @@ bool Timers::sRescheduleNeeded = false; #define TIMER_ID "timer " << mName << " (" << this << ") " -#define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ") " +#define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ")" typedef void (*sighandler_t)(int); @@ -232,11 +232,7 @@ if (timeToExpiry <= 0) { - /* - BOX_TRACE("timer " << *i << " has expired, " - "triggering it"); - */ - BOX_TRACE(TIMER_ID_OF(**i) "has expired, " + BOX_TRACE(TIMER_ID_OF(**i) " has expired, " "triggering " << BOX_FORMAT_MICROSECONDS(-timeToExpiry) << " late"); @@ -483,6 +479,30 @@ Stop(); } +void Timer::LogAssignment(const Timer &From) +{ + #ifndef BOX_RELEASE_BUILD + if (mExpired) + { + BOX_TRACE(TIMER_ID "initialised from timer " << + TIMER_ID_OF(From) << ", already expired, " + "will not fire"); + } + else if (mExpires == 0) + { + BOX_TRACE(TIMER_ID "initialised from timer " << + TIMER_ID_OF(From) << ", no expiry, " + "will not fire"); + } + else + { + BOX_TRACE(TIMER_ID "initialised from timer " << + TIMER_ID_OF(From) << ", to fire after " << + BOX_FORMAT_MICROSECONDS(From.mExpires)); + } + #endif +} + // -------------------------------------------------------------------------- // // Function @@ -502,25 +522,7 @@ , mTimerHandle(INVALID_HANDLE_VALUE) #endif { - #ifndef BOX_RELEASE_BUILD - if (mExpired) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "already expired, will not fire"); - } - else if (mExpires == 0) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "no expiry, will not fire"); - } - else - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "to fire at " << - (int)(mExpires / MICRO_SEC_IN_SEC_LL) << "." << - (int)(mExpires % MICRO_SEC_IN_SEC_LL)); - } - #endif + LogAssignment(rToCopy); if (!mExpired && mExpires != 0) { @@ -543,25 +545,7 @@ Timer& Timer::operator=(const Timer& rToCopy) { - #ifndef BOX_RELEASE_BUILD - if (rToCopy.mExpired) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "already expired, will not fire"); - } - else if (rToCopy.mExpires == 0) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "no expiry, will not fire"); - } - else - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "to fire at " << - (int)(rToCopy.mExpires / MICRO_SEC_IN_SEC_LL) << "." << - (int)(rToCopy.mExpires % MICRO_SEC_IN_SEC_LL)); - } - #endif + LogAssignment(rToCopy); Timers::Remove(*this); Stop(); Modified: box/trunk/lib/common/Timer.h =================================================================== --- box/trunk/lib/common/Timer.h 2012-11-07 00:08:14 UTC (rev 3131) +++ box/trunk/lib/common/Timer.h 2012-11-07 00:24:37 UTC (rev 3132) @@ -76,6 +76,7 @@ void Start(); void Start(int64_t timeoutMillis); void Stop(); + void LogAssignment(const Timer &From); #ifdef WIN32 HANDLE mTimerHandle; From subversion at boxbackup.org Wed Nov 7 00:33:38 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 7 Nov 2012 00:33:38 GMT Subject: [Box Backup-commit] COMMIT r3133 - box/trunk/lib/common Message-ID: <201211070033.qA70Xc50058537@wm.boxbackup.org> Author: chris Date: 2012-11-07 00:33:38 +0000 (Wed, 07 Nov 2012) New Revision: 3133 Modified: box/trunk/lib/common/Timer.cpp Log: Fix incorrect logging of timer assignment, and format expiry time as a human time. Modified: box/trunk/lib/common/Timer.cpp =================================================================== --- box/trunk/lib/common/Timer.cpp 2012-11-07 00:24:37 UTC (rev 3132) +++ box/trunk/lib/common/Timer.cpp 2012-11-07 00:33:38 UTC (rev 3133) @@ -482,13 +482,13 @@ void Timer::LogAssignment(const Timer &From) { #ifndef BOX_RELEASE_BUILD - if (mExpired) + if (From.mExpired) { BOX_TRACE(TIMER_ID "initialised from timer " << TIMER_ID_OF(From) << ", already expired, " "will not fire"); } - else if (mExpires == 0) + else if (From.mExpires == 0) { BOX_TRACE(TIMER_ID "initialised from timer " << TIMER_ID_OF(From) << ", no expiry, " @@ -497,8 +497,8 @@ else { BOX_TRACE(TIMER_ID "initialised from timer " << - TIMER_ID_OF(From) << ", to fire after " << - BOX_FORMAT_MICROSECONDS(From.mExpires)); + TIMER_ID_OF(From) << ", to fire at " << + FormatTime(From.mExpires, false, true)); } #endif } From subversion at boxbackup.org Wed Nov 7 19:59:44 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 7 Nov 2012 19:59:44 GMT Subject: [Box Backup-commit] COMMIT r3134 - in box/trunk: bin/bbackupd lib/backupstore Message-ID: <201211071959.qA7JxicD064814@wm.boxbackup.org> Author: chris Date: 2012-11-07 19:59:44 +0000 (Wed, 07 Nov 2012) New Revision: 3134 Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp box/trunk/bin/bbackupd/BackupClientContext.h box/trunk/lib/backupstore/BackupStoreFileDiff.cpp Log: Replace magic number 1000 with MILLI_SEC_IN_SEC when initialising timers. Add the start of ExperimentalSnapshotMode support. Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-11-07 00:33:38 UTC (rev 3133) +++ box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-11-07 19:59:44 UTC (rev 3134) @@ -53,26 +53,29 @@ ProgressNotifier& rProgressNotifier, bool TcpNiceMode ) - : mrResolver(rResolver), - mrTLSContext(rTLSContext), - mHostname(rHostname), - mPort(Port), - mAccountNumber(AccountNumber), - mExtendedLogging(ExtendedLogging), - mExtendedLogToFile(ExtendedLogToFile), - mExtendedLogFile(ExtendedLogFile), - mpExtendedLogFileHandle(NULL), - mClientStoreMarker(ClientStoreMarker_NotKnown), - mpDeleteList(0), - mpCurrentIDMap(0), - mpNewIDMap(0), - mStorageLimitExceeded(false), - mpExcludeFiles(0), - mpExcludeDirs(0), - mKeepAliveTimer(0, "KeepAliveTime"), - mbIsManaged(false), - mrProgressNotifier(rProgressNotifier), - mTcpNiceMode(TcpNiceMode) +: mExperimentalSnapshotMode(false), + mrResolver(rResolver), + mrTLSContext(rTLSContext), + mHostname(rHostname), + mPort(Port), + mAccountNumber(AccountNumber), + mpSocket(0), + mpConnection(0), + mExtendedLogging(ExtendedLogging), + mExtendedLogToFile(ExtendedLogToFile), + mExtendedLogFile(ExtendedLogFile), + mpExtendedLogFileHandle(NULL), + mClientStoreMarker(ClientStoreMarker_NotKnown), + mpDeleteList(0), + mpCurrentIDMap(0), + mpNewIDMap(0), + mStorageLimitExceeded(false), + mpExcludeFiles(0), + mpExcludeDirs(0), + mKeepAliveTimer(0, "KeepAliveTime"), + mbIsManaged(false), + mrProgressNotifier(rProgressNotifier) + mTcpNiceMode(TcpNiceMode) { } @@ -507,7 +510,8 @@ { mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); - mKeepAliveTimer = Timer(mKeepAliveTime * 1000, "KeepAliveTime"); + mKeepAliveTimer = Timer(mKeepAliveTime * MILLI_SEC_IN_SEC, + "KeepAliveTime"); } // -------------------------------------------------------------------------- @@ -567,7 +571,8 @@ BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); mapConnection->QueryGetIsAlive(); - mKeepAliveTimer = Timer(mKeepAliveTime * 1000, "KeepAliveTime"); + mKeepAliveTimer = Timer(mKeepAliveTime * MILLI_SEC_IN_SEC, + "KeepAliveTime"); } int BackupClientContext::GetMaximumDiffingTime() Modified: box/trunk/bin/bbackupd/BackupClientContext.h =================================================================== --- box/trunk/bin/bbackupd/BackupClientContext.h 2012-11-07 00:33:38 UTC (rev 3133) +++ box/trunk/bin/bbackupd/BackupClientContext.h 2012-11-07 19:59:44 UTC (rev 3134) @@ -218,6 +218,8 @@ } } + bool mExperimentalSnapshotMode; + private: LocationResolver &mrResolver; TLSContext &mrTLSContext; Modified: box/trunk/lib/backupstore/BackupStoreFileDiff.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreFileDiff.cpp 2012-11-07 00:33:38 UTC (rev 3133) +++ box/trunk/lib/backupstore/BackupStoreFileDiff.cpp 2012-11-07 19:59:44 UTC (rev 3134) @@ -469,8 +469,8 @@ if(pDiffTimer && pDiffTimer->IsManaged()) { - maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime() * 1000, - "MaximumDiffingTime"); + maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime() * + MILLI_SEC_IN_SEC, "MaximumDiffingTime"); } std::map goodnessOfFit; From subversion at boxbackup.org Wed Nov 7 23:14:01 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 7 Nov 2012 23:14:01 GMT Subject: [Box Backup-commit] COMMIT r3135 - box/trunk/lib/backupstore Message-ID: <201211072314.qA7NE1xN065757@wm.boxbackup.org> Author: chris Date: 2012-11-07 23:14:01 +0000 (Wed, 07 Nov 2012) New Revision: 3135 Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp Log: Remove commented-out method copied from BackupStoreInfo and not used. Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp 2012-11-07 19:59:44 UTC (rev 3134) +++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp 2012-11-07 23:14:01 UTC (rev 3135) @@ -162,89 +162,6 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreRefCountDatabase::Save() -// Purpose: Save modified info back to disc -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -/* -void BackupStoreRefCountDatabase::Save() -{ - // Make sure we're initialised (although should never come to this) - if(mFilename.empty() || mAccount.GetID() == 0) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Can we do this? - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - - // Then... open a write file - RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); - rf.Open(true); // allow overwriting - - // Make header - info_StreamFormat hdr; - hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); - hdr.mAccountID = htonl(mAccountID); - hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); - hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); - hdr.mBlocksUsed = box_hton64(mBlocksUsed); - hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); - hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); - hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); - hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); - hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); - hdr.mCurrentMarkNumber = 0; - hdr.mOptionsPresent = 0; - hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); - - // Write header - rf.Write(&hdr, sizeof(hdr)); - - // Write the deleted object list - if(mDeletedDirectories.size() > 0) - { - int64_t objs[NUM_DELETED_DIRS_BLOCK]; - - int tosave = mDeletedDirectories.size(); - std::vector::iterator i(mDeletedDirectories.begin()); - while(tosave > 0) - { - // How many in this one? - int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); - - // Add them - for(int t = 0; t < b; ++t) - { - ASSERT(i != mDeletedDirectories.end()); - objs[t] = box_hton64((*i)); - i++; - } - - // Write - rf.Write(objs, b * sizeof(int64_t)); - - // Number saved - tosave -= b; - } - } - - // Commit it to disc, converting it to RAID now - rf.Commit(true); - - // Mark is as not modified - mIsModified = false; -} -*/ - - -// -------------------------------------------------------------------------- -// -// Function // Name: BackupStoreRefCountDatabase::GetRefCount(int64_t // ObjectID) // Purpose: Get the number of references to the specified object From subversion at boxbackup.org Thu Nov 8 00:33:20 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 00:33:20 GMT Subject: [Box Backup-commit] COMMIT r3136 - box/trunk/lib/common Message-ID: <201211080033.qA80XKHu066150@wm.boxbackup.org> Author: chris Date: 2012-11-08 00:33:20 +0000 (Thu, 08 Nov 2012) New Revision: 3136 Modified: box/trunk/lib/common/MemBlockStream.cpp box/trunk/lib/common/MemBlockStream.h Log: Allow constructing an empty MemBlockStream, as it's useful for tests. Modified: box/trunk/lib/common/MemBlockStream.cpp =================================================================== --- box/trunk/lib/common/MemBlockStream.cpp 2012-11-07 23:14:01 UTC (rev 3135) +++ box/trunk/lib/common/MemBlockStream.cpp 2012-11-08 00:33:20 UTC (rev 3136) @@ -22,6 +22,20 @@ // // Function // Name: MemBlockStream::MemBlockStream() +// Purpose: Constructor with no contents +// Created: 2012/11/07 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream() +: mpBuffer(NULL), + mBytesInBuffer(0), + mReadPosition(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream() // Purpose: Constructor (doesn't copy block, careful with lifetimes) // Created: 2003/09/05 // @@ -69,7 +83,6 @@ ASSERT(mBytesInBuffer >= 0); } - // -------------------------------------------------------------------------- // // Function Modified: box/trunk/lib/common/MemBlockStream.h =================================================================== --- box/trunk/lib/common/MemBlockStream.h 2012-11-07 23:14:01 UTC (rev 3135) +++ box/trunk/lib/common/MemBlockStream.h 2012-11-08 00:33:20 UTC (rev 3136) @@ -27,6 +27,7 @@ class MemBlockStream : public IOStream { public: + MemBlockStream(); MemBlockStream(const void *pBuffer, int Size); MemBlockStream(const StreamableMemBlock &rBlock); MemBlockStream(const CollectInBufferStream &rBuffer); From subversion at boxbackup.org Thu Nov 8 00:35:06 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 00:35:06 GMT Subject: [Box Backup-commit] COMMIT r3137 - box/trunk/lib/backupstore Message-ID: <201211080035.qA80Z690066173@wm.boxbackup.org> Author: chris Date: 2012-11-08 00:35:06 +0000 (Thu, 08 Nov 2012) New Revision: 3137 Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp box/trunk/lib/backupstore/BackupStoreInfo.cpp box/trunk/lib/backupstore/BackupStoreInfo.h Log: Make CreateForRegeneration save the AccountEnabled flag and any extra data copied from the original info file. Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreCheck2.cpp 2012-11-08 00:33:20 UTC (rev 3136) +++ box/trunk/lib/backupstore/BackupStoreCheck2.cpp 2012-11-08 00:35:06 UTC (rev 3137) @@ -646,6 +646,15 @@ } // Build a new store info + std::auto_ptr extra_data; + if(pOldInfo.get()) + { + extra_data.reset(new MemBlockStream(pOldInfo->GetExtraData())); + } + else + { + extra_data.reset(new MemBlockStream(/* empty */)); + } std::auto_ptr info(BackupStoreInfo::CreateForRegeneration( mAccountID, mAccountName, @@ -658,7 +667,9 @@ mBlocksInDeletedFiles, mBlocksInDirectories, softLimit, - hardLimit)); + hardLimit, + (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), + *extra_data)); info->AdjustNumFiles(mNumFiles); info->AdjustNumOldFiles(mNumOldFiles); info->AdjustNumDeletedFiles(mNumDeletedFiles); Modified: box/trunk/lib/backupstore/BackupStoreInfo.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreInfo.cpp 2012-11-08 00:33:20 UTC (rev 3136) +++ box/trunk/lib/backupstore/BackupStoreInfo.cpp 2012-11-08 00:35:06 UTC (rev 3137) @@ -284,7 +284,8 @@ int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInCurrentFiles, int64_t BlocksInOldFiles, int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, - int64_t BlockSoftLimit, int64_t BlockHardLimit) + int64_t BlockSoftLimit, int64_t BlockHardLimit, + bool AccountEnabled, IOStream& ExtraData) { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); @@ -298,7 +299,7 @@ info->mDiscSet = DiscSet; info->mFilename = fn; info->mReadOnly = false; - + // Insert info starting info info->mClientStoreMarker = 0; info->mLastObjectIDUsed = LastObjectID; @@ -309,7 +310,11 @@ info->mBlocksInDirectories = BlocksInDirectories; info->mBlocksSoftLimit = BlockSoftLimit; info->mBlocksHardLimit = BlockHardLimit; + info->mAccountEnabled = AccountEnabled; + ExtraData.CopyStreamTo(info->mExtraData); + info->mExtraData.SetForReading(); + // return it to caller return info; } Modified: box/trunk/lib/backupstore/BackupStoreInfo.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreInfo.h 2012-11-08 00:33:20 UTC (rev 3136) +++ box/trunk/lib/backupstore/BackupStoreInfo.h 2012-11-08 00:35:06 UTC (rev 3137) @@ -139,15 +139,20 @@ const CollectInBufferStream& GetExtraData() const { return mExtraData; } void SetAccountEnabled(bool IsEnabled) {mAccountEnabled = IsEnabled; } -private: + /** + * @return a new BackupStoreInfo with the requested properties. + * This is exposed to allow testing, do not use otherwise! + */ static std::auto_ptr CreateForRegeneration( int32_t AccountID, const std::string &rAccountName, const std::string &rRootDir, int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInCurrentFiles, int64_t BlocksInOldFiles, int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, - int64_t BlockSoftLimit, int64_t BlockHardLimit); + int64_t BlockSoftLimit, int64_t BlockHardLimit, + bool AccountEnabled, IOStream& ExtraData); +private: // Location information // Be VERY careful about changing types of these values, as // they now define the sizes of fields on disk (via Archive). From subversion at boxbackup.org Thu Nov 8 00:35:47 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 00:35:47 GMT Subject: [Box Backup-commit] COMMIT r3138 - box/trunk/test/backupstore Message-ID: <201211080035.qA80ZlkR066191@wm.boxbackup.org> Author: chris Date: 2012-11-08 00:35:47 +0000 (Thu, 08 Nov 2012) New Revision: 3138 Modified: box/trunk/test/backupstore/testbackupstore.cpp Log: Test that BackupStoreInfo::CreateForRegeneration saves AccountEnabled flag and ExtraData. Modified: box/trunk/test/backupstore/testbackupstore.cpp =================================================================== --- box/trunk/test/backupstore/testbackupstore.cpp 2012-11-08 00:35:06 UTC (rev 3137) +++ box/trunk/test/backupstore/testbackupstore.cpp 2012-11-08 00:35:47 UTC (rev 3138) @@ -2226,7 +2226,10 @@ ( const std::string& test_phase, const info_StreamFormat_1& expected, - const BackupStoreInfo& actual + const BackupStoreInfo& actual, + const std::string& expected_account_name, + bool expected_account_enabled, + const MemBlockStream& extra_data = MemBlockStream(/* empty */) ) { TEST_EQUAL_LINE(ntohl(expected.mAccountID), actual.GetAccountID(), @@ -2242,6 +2245,8 @@ TEST_INFO_EQUAL(BlocksInDirectories); TEST_INFO_EQUAL(BlocksSoftLimit); TEST_INFO_EQUAL(BlocksHardLimit); + #undef TEST_INFO_EQUAL + // These attributes of the v2 structure are not supported by v1, // so they should all be initialised to 0 TEST_EQUAL_LINE(0, actual.GetBlocksInCurrentFiles(), @@ -2252,18 +2257,33 @@ test_phase << " NumDeletedFiles"); TEST_EQUAL_LINE(0, actual.GetNumDirectories(), test_phase << " NumDirectories"); + // These attributes of the old v1 structure are not actually loaded // or used: // TEST_INFO_EQUAL(CurrentMarkNumber); // TEST_INFO_EQUAL(OptionsPresent); + TEST_EQUAL_LINE(box_ntoh64(expected.mNumberDeletedDirectories), actual.GetDeletedDirectories().size(), test_phase << " number of deleted directories"); - TEST_EQUAL_LINE(13, actual.GetDeletedDirectories()[0], - test_phase << " deleted directory 1"); - TEST_EQUAL_LINE(14, actual.GetDeletedDirectories()[1], - test_phase << " deleted directory 2"); - #undef TEST_INFO_EQUAL + + for (size_t i = 0; i < box_ntoh64(expected.mNumberDeletedDirectories) && + i < actual.GetDeletedDirectories().size(); i++) + { + TEST_EQUAL_LINE(13 + i, actual.GetDeletedDirectories()[i], + test_phase << " deleted directory " << (i+1)); + } + + TEST_EQUAL_LINE(expected_account_name, actual.GetAccountName(), + test_phase << " AccountName"); + TEST_EQUAL_LINE(expected_account_enabled, actual.IsAccountEnabled(), + test_phase << " AccountEnabled"); + + TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), + test_phase << " extra data has wrong size"); + TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), + actual.GetExtraData().GetBuffer(), extra_data.GetSize()), + test_phase << " extra data has wrong contents"); } int test_read_old_backupstoreinfo_files() @@ -2305,11 +2325,8 @@ std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, - *apInfo); - TEST_EQUAL_LINE("", apInfo->GetAccountName(), - "account loaded from version 1 format should have no name"); - TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), - "account loaded from version 1 format should be enabled by default"); + *apInfo, "" /* no name by default */, + true /* enabled by default */); apInfo->SetAccountName("bonk"); @@ -2329,14 +2346,11 @@ "format version in newly saved BackupStoreInfo"); rfr.reset(); + // load it, and check that all values are loaded properly apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded in v1, resaved in v2", - info_v1, *apInfo); - TEST_EQUAL_LINE("bonk", apInfo->GetAccountName(), - "account loaded from version 1 format should have no name"); - TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), - "account resaved in v2 format should preserve AccountEnabled"); + info_v1, *apInfo, "bonk", true); // Check that the new AccountEnabled flag is saved properly apInfo->SetAccountEnabled(false); @@ -2344,16 +2358,15 @@ apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2, loaded in v2", - info_v1, *apInfo); - TEST_EQUAL_LINE(false, apInfo->IsAccountEnabled(), - "AccountEnabled flag was set to false but not loaded correctly"); + info_v1, *apInfo, "bonk", false /* as modified above */); apInfo->SetAccountEnabled(true); apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ true); - TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), - "AccountEnabled flag was set to true but not loaded correctly"); - + compare_backupstoreinfo_values_to_expected("resaved in v2 with " + "account enabled", info_v1, *apInfo, "bonk", + true /* as modified above */); + // Now save the info in v2 format without the AccountEnabled flag // (boxbackup 0.11 format) and check that the flag is set to true // for backwards compatibility @@ -2387,11 +2400,9 @@ apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2 without " - "AccountEnabled", info_v1, *apInfo); - TEST_EQUAL_LINE("test", apInfo->GetAccountName(), - "account loaded from short v2 format should preserve AccountName"); - TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), - "default for missing AccountEnabled should be true"); + "AccountEnabled", info_v1, *apInfo, "test", true); + // Default for missing AccountEnabled should be true + // Rewrite using full length, so that the first 4 bytes of extra data // doesn't get swallowed by "extra data". apInfo->Save(/* allowOverwrite */ true); @@ -2423,11 +2434,8 @@ // Save the file and load again, check that the extra data is still there apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); - TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), - "wrong amount of extra data saved and reloaded from info file"); - TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), - apInfo->GetExtraData().GetBuffer(), extra_data.GetSize()), - "extra data saved and reloaded from info file has wrong contents"); + compare_backupstoreinfo_values_to_expected("saved in future format " + "with extra_data", info_v1, *apInfo, "test", true, extra_data); // Check that the new bbstoreaccounts command sets the flag properly TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS @@ -2443,6 +2451,39 @@ TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled yes' should have set AccountEnabled flag"); + // Check that BackupStoreInfo::CreateForRegeneration saves all the + // expected properties, including any extra data for forward + // compatibility + extra_data.Seek(0, IOStream::SeekType_Absolute); + apInfo = BackupStoreInfo::CreateForRegeneration( + apInfo->GetAccountID(), "spurtle" /* rAccountName */, + "backup/01234567/" /* rRootDir */, 0 /* DiscSet */, + apInfo->GetLastObjectIDUsed(), + apInfo->GetBlocksUsed(), + apInfo->GetBlocksInCurrentFiles(), + apInfo->GetBlocksInOldFiles(), + apInfo->GetBlocksInDeletedFiles(), + apInfo->GetBlocksInDirectories(), + apInfo->GetBlocksSoftLimit(), + apInfo->GetBlocksHardLimit(), + false /* AccountEnabled */, + extra_data); + // CreateForRegeneration always sets the ClientStoreMarker to 0 + info_v1.mClientStoreMarker = 0; + // CreateForRegeneration does not store any deleted directories + info_v1.mNumberDeletedDirectories = 0; + + // check that the store info has the correct values in memory + compare_backupstoreinfo_values_to_expected("stored by " + "BackupStoreInfo::CreateForRegeneration", info_v1, *apInfo, + "spurtle", false /* AccountEnabled */, extra_data); + // Save the file and load again, check that the extra data is still there + apInfo->Save(/* allowOverwrite */ true); + apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); + compare_backupstoreinfo_values_to_expected("saved by " + "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, + *apInfo, "spurtle", false /* AccountEnabled */, extra_data); + // Delete the account to leave the store in the same state as before TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf delete 01234567 yes") == 0); From subversion at boxbackup.org Thu Nov 8 00:46:13 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 00:46:13 GMT Subject: [Box Backup-commit] COMMIT r3139 - box/trunk/lib/backupstore Message-ID: <201211080046.qA80kDFY066256@wm.boxbackup.org> Author: chris Date: 2012-11-08 00:46:13 +0000 (Thu, 08 Nov 2012) New Revision: 3139 Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp Log: Fix missing #include for MemBlockStream.h Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreCheck2.cpp 2012-11-08 00:35:47 UTC (rev 3138) +++ box/trunk/lib/backupstore/BackupStoreCheck2.cpp 2012-11-08 00:46:13 UTC (rev 3139) @@ -12,17 +12,18 @@ #include #include +#include "autogen_BackupStoreException.h" #include "BackupStoreCheck.h" -#include "StoreStructure.h" -#include "RaidFileRead.h" -#include "RaidFileWrite.h" -#include "autogen_BackupStoreException.h" -#include "BackupStoreObjectMagic.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" #include "BackupStoreFile.h" #include "BackupStoreFileWire.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreConstants.h" #include "BackupStoreInfo.h" +#include "BackupStoreObjectMagic.h" +#include "MemBlockStream.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "StoreStructure.h" #include "MemLeakFindOn.h" From subversion at boxbackup.org Thu Nov 8 00:46:49 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 00:46:49 GMT Subject: [Box Backup-commit] COMMIT r3140 - box/trunk/test/backupstorefix/testfiles Message-ID: <201211080046.qA80knYl066274@wm.boxbackup.org> Author: chris Date: 2012-11-08 00:46:49 +0000 (Thu, 08 Nov 2012) New Revision: 3140 Modified: box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in Log: Fix additional space before [FILENAME NOT ENCRYPTED] causing test failures. Modified: box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in =================================================================== --- box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in 2012-11-08 00:46:13 UTC (rev 3139) +++ box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in 2012-11-08 00:46:49 UTC (rev 3140) @@ -104,7 +104,7 @@ { print LISTING_COPY; chomp; s/\r//; - s/\[FILENAME NOT ENCRYPTED\]//; + s/ \[FILENAME NOT ENCRYPTED\]//; if(exists $expected{$_}) { delete $expected{$_} From subversion at boxbackup.org Thu Nov 8 00:49:38 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 00:49:38 GMT Subject: [Box Backup-commit] COMMIT r3141 - box/trunk/bin/bbackupd Message-ID: <201211080049.qA80ncEU066298@wm.boxbackup.org> Author: chris Date: 2012-11-08 00:49:38 +0000 (Thu, 08 Nov 2012) New Revision: 3141 Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp Log: Fix compile failure due to removal of fields from BackupClientContext. Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-11-08 00:46:49 UTC (rev 3140) +++ box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-11-08 00:49:38 UTC (rev 3141) @@ -59,8 +59,6 @@ mHostname(rHostname), mPort(Port), mAccountNumber(AccountNumber), - mpSocket(0), - mpConnection(0), mExtendedLogging(ExtendedLogging), mExtendedLogToFile(ExtendedLogToFile), mExtendedLogFile(ExtendedLogFile), @@ -74,7 +72,7 @@ mpExcludeDirs(0), mKeepAliveTimer(0, "KeepAliveTime"), mbIsManaged(false), - mrProgressNotifier(rProgressNotifier) + mrProgressNotifier(rProgressNotifier), mTcpNiceMode(TcpNiceMode) { } From subversion at boxbackup.org Thu Nov 8 20:44:25 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 20:44:25 GMT Subject: [Box Backup-commit] COMMIT r3142 - in box/trunk: lib/backupstore test/backupstore Message-ID: <201211082044.qA8KiP34072771@wm.boxbackup.org> Author: chris Date: 2012-11-08 20:44:25 +0000 (Thu, 08 Nov 2012) New Revision: 3142 Modified: box/trunk/lib/backupstore/BackupStoreInfo.cpp box/trunk/test/backupstore/testbackupstore.cpp Log: Ensure that newly created accounts are enabled by default. Modified: box/trunk/lib/backupstore/BackupStoreInfo.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreInfo.cpp 2012-11-08 00:49:38 UTC (rev 3141) +++ box/trunk/lib/backupstore/BackupStoreInfo.cpp 2012-11-08 20:44:25 UTC (rev 3142) @@ -48,7 +48,8 @@ mNumFiles(0), mNumOldFiles(0), mNumDeletedFiles(0), - mNumDirectories(0) + mNumDirectories(0), + mAccountEnabled(true) { } Modified: box/trunk/test/backupstore/testbackupstore.cpp =================================================================== --- box/trunk/test/backupstore/testbackupstore.cpp 2012-11-08 00:49:38 UTC (rev 3141) +++ box/trunk/test/backupstore/testbackupstore.cpp 2012-11-08 20:44:25 UTC (rev 3142) @@ -435,7 +435,7 @@ void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) { - printf("Test for del: %llx\n", DirID); + printf("Test for del: %llx\n", (unsigned long long)DirID); // Command std::auto_ptr dirreply(protocol.QueryListDirectory( @@ -476,7 +476,7 @@ void set_refcount(int64_t ObjectID, uint32_t RefCount = 1) { - if (ExpectedRefCounts.size() <= ObjectID); + if ((int64_t)ExpectedRefCounts.size() <= ObjectID); { ExpectedRefCounts.resize(ObjectID + 1, 0); } @@ -519,7 +519,8 @@ subdirid = dirCreate->GetObjectID(); } - printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid); + printf("Create subdirs, depth = %d, dirid = %llx\n", depth, + (unsigned long long)subdirid); TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) @@ -2293,6 +2294,10 @@ " -c testfiles/bbstored.conf create 01234567 0 " "10000B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, + "backup/01234567/", 0, /* ReadOnly */ false); + TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), + "'bbstoreaccounts create' should have set AccountEnabled flag"); info_StreamFormat_1 info_v1; info_v1.mMagicValue = htonl(INFO_MAGIC_VALUE_1); @@ -2322,8 +2327,8 @@ rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, - "backup/01234567/", 0, /* ReadOnly */ false); + apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, + /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, *apInfo, "" /* no name by default */, true /* enabled by default */); From subversion at boxbackup.org Thu Nov 8 20:44:54 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 20:44:54 GMT Subject: [Box Backup-commit] COMMIT r3143 - box/trunk/infrastructure Message-ID: <201211082044.qA8KisRo072788@wm.boxbackup.org> Author: chris Date: 2012-11-08 20:44:53 +0000 (Thu, 08 Nov 2012) New Revision: 3143 Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp Log: Check for /dev/log and don't report if fds to it are leaked. Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp =================================================================== --- box/trunk/infrastructure/buildenv-testmain-template.cpp 2012-11-08 20:44:25 UTC (rev 3142) +++ box/trunk/infrastructure/buildenv-testmain-template.cpp 2012-11-08 20:44:53 UTC (rev 3143) @@ -29,8 +29,10 @@ #include #endif +#include #include #include +#include #include #include @@ -76,8 +78,16 @@ #define FILEDES_MAX 256 -bool filedes_open[FILEDES_MAX]; +typedef enum +{ + OPEN, + CLOSED, + SYSLOG +} +filedes_t; +filedes_t filedes_open[FILEDES_MAX]; + bool check_filedes(bool report) { bool allOk = true; @@ -87,12 +97,50 @@ { if(::fcntl(d, F_GETFD) != -1) { - // File descriptor obviously exists - if (report && !filedes_open[d]) + // File descriptor obviously exists, but is it /dev/log? + + struct stat st; + bool stat_success = false; + bool is_syslog_socket = false; + + if(fstat(d, &st) == 0) { - struct stat st; - if (fstat(d, &st) == 0) + stat_success = true; + } + + if(stat_success && (st.st_mode & S_IFSOCK)) + { + char buffer[256]; + socklen_t addrlen = sizeof(buffer); + + if(getpeername(d, (sockaddr*)buffer, &addrlen) != 0) { + BOX_WARNING("Failed to getpeername(" << + d << "), cannot identify /dev/log"); + } + else + { + struct sockaddr_un *sa = + (struct sockaddr_un *)buffer; + if(sa->sun_family == PF_UNIX && + !strcmp(sa->sun_path, "/dev/log")) + { + is_syslog_socket = true; + } + } + } + + if(report && filedes_open[d] != OPEN) + { + if(filedes_open[d] == SYSLOG) + { + // Different libcs have different ideas + // about when to open and close this + // socket, and it's not a leak, so + // ignore it. + } + else if(stat_success) + { int m = st.st_mode; #define flag(x) ((m & x) ? #x " " : "") BOX_FATAL("File descriptor " << d << @@ -113,24 +161,33 @@ } allOk = false; - } else if (!report) { - filedes_open[d] = true; + filedes_open[d] = is_syslog_socket ? SYSLOG : OPEN; } } else { - if (report && filedes_open[d]) + if (report && filedes_open[d] != CLOSED) { - BOX_FATAL("File descriptor " << d << - " was open, now closed"); - allOk = false; + if (filedes_open[d] == SYSLOG) + { + // Different libcs have different ideas + // about when to open and close this + // socket, and it's not a leak, so + // ignore it. + } + else if(filedes_open[d] == OPEN) + { + BOX_FATAL("File descriptor " << d << + " was open, now closed"); + allOk = false; + } } else { - filedes_open[d] = false; + filedes_open[d] = CLOSED; } } } From subversion at boxbackup.org Thu Nov 8 20:45:39 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 20:45:39 GMT Subject: [Box Backup-commit] COMMIT r3144 - in box/trunk/test: backupstore bbackupd Message-ID: <201211082045.qA8KjdUx072809@wm.boxbackup.org> Author: chris Date: 2012-11-08 20:45:39 +0000 (Thu, 08 Nov 2012) New Revision: 3144 Modified: box/trunk/test/backupstore/Makefile.extra box/trunk/test/bbackupd/Makefile.extra Log: Link to HousekeepStoreAccount.o in the right directory since the big refactor. Modified: box/trunk/test/backupstore/Makefile.extra =================================================================== --- box/trunk/test/backupstore/Makefile.extra 2012-11-08 20:44:53 UTC (rev 3143) +++ box/trunk/test/backupstore/Makefile.extra 2012-11-08 20:45:39 UTC (rev 3144) @@ -1 +1 @@ -link-extra: ../../bin/bbstored/HousekeepStoreAccount.o +link-extra: ../../lib/backupstore/HousekeepStoreAccount.o Modified: box/trunk/test/bbackupd/Makefile.extra =================================================================== --- box/trunk/test/bbackupd/Makefile.extra 2012-11-08 20:44:53 UTC (rev 3143) +++ box/trunk/test/bbackupd/Makefile.extra 2012-11-08 20:45:39 UTC (rev 3144) @@ -7,7 +7,7 @@ ../../bin/bbackupd/Win32ServiceFunctions.o \ ../../bin/bbackupd/BackupDaemon.o \ ../../bin/bbstored/BBStoreDHousekeeping.o \ - ../../bin/bbstored/HousekeepStoreAccount.o \ + ../../lib/backupstore/HousekeepStoreAccount.o \ ../../lib/backupstore/autogen_BackupProtocol.o \ ../../lib/backupstore/BackupStoreContext.o \ ../../lib/backupstore/BackupCommands.o \ From subversion at boxbackup.org Thu Nov 8 22:05:12 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 8 Nov 2012 22:05:12 GMT Subject: [Box Backup-commit] COMMIT r3145 - box/trunk/infrastructure Message-ID: <201211082205.qA8M5CmL073198@wm.boxbackup.org> Author: chris Date: 2012-11-08 22:05:12 +0000 (Thu, 08 Nov 2012) New Revision: 3145 Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp Log: Don't flag an error if the only sockets "leaked" are syslog sockets. Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp =================================================================== --- box/trunk/infrastructure/buildenv-testmain-template.cpp 2012-11-08 20:45:39 UTC (rev 3144) +++ box/trunk/infrastructure/buildenv-testmain-template.cpp 2012-11-08 22:05:12 UTC (rev 3145) @@ -153,14 +153,14 @@ flag(S_IFLNK) << flag(S_IFSOCK) << " or " << m << ")"); + allOk = false; } else { BOX_FATAL("File descriptor " << d << " left open (and stat failed)"); + allOk = false; } - - allOk = false; } else if (!report) { From subversion at boxbackup.org Wed Nov 14 23:45:13 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 14 Nov 2012 23:45:13 GMT Subject: [Box Backup-commit] COMMIT r3146 - box/trunk/infrastructure/mingw Message-ID: <201211142345.qAENjD0q019400@wm.boxbackup.org> Author: chris Date: 2012-11-14 23:45:13 +0000 (Wed, 14 Nov 2012) New Revision: 3146 Modified: box/trunk/infrastructure/mingw/configure.sh Log: Request linker to statically link libz.a, since the default now appears to be dynamic linking, and that stops the tests from working since they can't find zlib-1.dll in the PATH. Remove redundant LIBS that should already be detected and used by the m4 configury. Modified: box/trunk/infrastructure/mingw/configure.sh =================================================================== --- box/trunk/infrastructure/mingw/configure.sh 2012-11-08 22:05:12 UTC (rev 3145) +++ box/trunk/infrastructure/mingw/configure.sh 2012-11-14 23:45:13 UTC (rev 3146) @@ -34,8 +34,7 @@ CFLAGS="-mno-cygwin -mthreads" \ CPPFLAGS="-mno-cygwin" \ CXXFLAGS="-mno-cygwin -mthreads" \ - LDFLAGS="-mno-cygwin -mthreads -L${DEP_PATH}/lib -L${LIBZ_PATH}" \ - LIBS="-lcrypto -lws2_32 -lgdi32" + LDFLAGS="-Wl,-Bstatic -mno-cygwin -mthreads -L${DEP_PATH}/lib -L${LIBZ_PATH}" then echo "Error: configure failed, aborting." >&2 exit 1 From subversion at boxbackup.org Wed Nov 14 23:47:55 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 14 Nov 2012 23:47:55 GMT Subject: [Box Backup-commit] COMMIT r3147 - box/trunk/infrastructure/m4 Message-ID: <201211142347.qAENlt3m019424@wm.boxbackup.org> Author: chris Date: 2012-11-14 23:47:55 +0000 (Wed, 14 Nov 2012) New Revision: 3147 Modified: box/trunk/infrastructure/m4/boxbackup_tests.m4 Log: Use AC_SEARCH_LIBS instead of AC_CHECK_LIB to search for EVP_CipherInit_ex, as it should already be in LIBS by this point and we don't really want to add a duplicate if we don't need it. http://www.gnu.org/savannah-checkouts/gnu/autoconf/manual/autoconf-2.69/html_node/Libraries.html also recommends the use of AC_SEARCH_LIBS instead. Check whether we have getpeername() and sys/un.h to fix compile errors on Windows with /dev/log fd leak test. Modified: box/trunk/infrastructure/m4/boxbackup_tests.m4 =================================================================== --- box/trunk/infrastructure/m4/boxbackup_tests.m4 2012-11-14 23:45:13 UTC (rev 3146) +++ box/trunk/infrastructure/m4/boxbackup_tests.m4 2012-11-14 23:47:55 UTC (rev 3147) @@ -109,9 +109,9 @@ [old-ssl], [AC_HELP_STRING([--enable-old-ssl], [Allow use of pre-0.9.7 Open SSL - NOT RECOMMENDED, read the documentation])]) -AC_CHECK_LIB( - [crypto], - [EVP_CipherInit_ex],, [ +AC_SEARCH_LIBS( + [EVP_CipherInit_ex], + [crypto],, [ if test "x$enable_old_ssl" = "xyes"; then AC_DEFINE([HAVE_OLD_SSL], 1, [Define to 1 if SSL is pre-0.9.7]) else @@ -136,8 +136,8 @@ AC_CHECK_HEADERS([dlfcn.h fcntl.h getopt.h process.h pwd.h signal.h]) AC_CHECK_HEADERS([syslog.h time.h cxxabi.h]) AC_CHECK_HEADERS([netinet/in.h netinet/tcp.h]) -AC_CHECK_HEADERS([sys/file.h sys/param.h sys/socket.h sys/time.h sys/types.h sys/wait.h]) -AC_CHECK_HEADERS([sys/uio.h sys/xattr.h]) +AC_CHECK_HEADERS([sys/file.h sys/param.h sys/socket.h sys/time.h sys/types.h]) +AC_CHECK_HEADERS([sys/uio.h sys/un.h sys/wait.h sys/xattr.h]) AC_CHECK_HEADERS([bsd/unistd.h]) AC_CHECK_HEADERS([sys/socket.h], [have_sys_socket_h=yes]) AC_CHECK_HEADERS([winsock2.h], [have_winsock2_h=yes]) @@ -254,7 +254,7 @@ AC_FUNC_ERROR_AT_LINE AC_TYPE_SIGNAL AC_FUNC_STAT -AC_CHECK_FUNCS([getpeereid lchown setproctitle getpid gettimeofday waitpid ftruncate]) +AC_CHECK_FUNCS([getpeereid getpeername lchown setproctitle getpid gettimeofday waitpid ftruncate]) AC_SEARCH_LIBS([setproctitle], ["bsd"]) # NetBSD implements kqueue too differently for us to get it fixed by 0.10 From subversion at boxbackup.org Wed Nov 14 23:48:51 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 14 Nov 2012 23:48:51 GMT Subject: [Box Backup-commit] COMMIT r3148 - box/trunk/infrastructure/m4 Message-ID: <201211142348.qAENmpH8019443@wm.boxbackup.org> Author: chris Date: 2012-11-14 23:48:50 +0000 (Wed, 14 Nov 2012) New Revision: 3148 Modified: box/trunk/infrastructure/m4/ax_check_ssl.m4 Log: Use AC_SEARCH_LIBS instead of AC_CHECK_LIB when searching for SSL libraries, to avoid adding duplicate libraries to LIBS. Modified: box/trunk/infrastructure/m4/ax_check_ssl.m4 =================================================================== --- box/trunk/infrastructure/m4/ax_check_ssl.m4 2012-11-14 23:47:55 UTC (rev 3147) +++ box/trunk/infrastructure/m4/ax_check_ssl.m4 2012-11-14 23:48:50 UTC (rev 3148) @@ -27,8 +27,8 @@ ax_check_ssl_found=yes AC_CHECK_HEADERS([openssl/ssl.h],, [ax_check_ssl_found=no]) - AC_CHECK_LIB([crypto], [HMAC_CTX_init]) - AC_CHECK_LIB([ssl], [SSL_read],, [ax_check_ssl_found=no]) + AC_SEARCH_LIBS([HMAC_CTX_init], [crypto]) + AC_SEARCH_LIBS([SSL_read], [ssl],, [ax_check_ssl_found=no]) if test "x$ax_check_ssl_found" = "xyes"; then AC_DEFINE([HAVE_SSL], 1, [Define to 1 if SSL is available]) From subversion at boxbackup.org Wed Nov 14 23:49:51 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 14 Nov 2012 23:49:51 GMT Subject: [Box Backup-commit] COMMIT r3149 - box/trunk/infrastructure Message-ID: <201211142349.qAENnpaa019462@wm.boxbackup.org> Author: chris Date: 2012-11-14 23:49:51 +0000 (Wed, 14 Nov 2012) New Revision: 3149 Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp Log: Don't use getpeername(), sys/socket.h or sys/un.h on platforms that don't include them, to fix compile on Windows. Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp =================================================================== --- box/trunk/infrastructure/buildenv-testmain-template.cpp 2012-11-14 23:48:50 UTC (rev 3148) +++ box/trunk/infrastructure/buildenv-testmain-template.cpp 2012-11-14 23:49:51 UTC (rev 3149) @@ -29,11 +29,17 @@ #include #endif -#include +#ifdef HAVE_SYS_SOCKET_H +# include +#endif + #include #include -#include +#ifdef HAVE_SYS_UN_H +# include +#endif + #include #include @@ -113,6 +119,7 @@ char buffer[256]; socklen_t addrlen = sizeof(buffer); +#ifdef HAVE_GETPEERNAME if(getpeername(d, (sockaddr*)buffer, &addrlen) != 0) { BOX_WARNING("Failed to getpeername(" << @@ -128,6 +135,7 @@ is_syslog_socket = true; } } +#endif // HAVE_GETPEERNAME } if(report && filedes_open[d] != OPEN) From subversion at boxbackup.org Thu Nov 15 00:00:06 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 15 Nov 2012 00:00:06 GMT Subject: [Box Backup-commit] COMMIT r3150 - box/trunk/infrastructure Message-ID: <201211150000.qAF006Yh019540@wm.boxbackup.org> Author: chris Date: 2012-11-15 00:00:06 +0000 (Thu, 15 Nov 2012) New Revision: 3150 Modified: box/trunk/infrastructure/makeparcels.pl.in Log: We need rm -f to remove read-only files from .svn directory copies in testfiles. Modified: box/trunk/infrastructure/makeparcels.pl.in =================================================================== --- box/trunk/infrastructure/makeparcels.pl.in 2012-11-14 23:49:51 UTC (rev 3149) +++ box/trunk/infrastructure/makeparcels.pl.in 2012-11-15 00:00:06 UTC (rev 3150) @@ -375,8 +375,8 @@ } else { - print MAKE "\tfind release debug -type f -exec rm {} \\;\n"; - print MAKE "\tfind . -name 'autogen_*' -type f -exec rm {} \\;\n"; + print MAKE "\tfind release debug -type f -exec rm -f {} \\;\n"; + print MAKE "\tfind . -name 'autogen_*' -type f -exec rm -f {} \\;\n"; } for my $parcel (@parcels) From subversion at boxbackup.org Sat Nov 17 00:54:05 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 17 Nov 2012 00:54:05 GMT Subject: [Box Backup-commit] COMMIT r3151 - box/trunk/lib/common Message-ID: <201211170054.qAH0s58J034949@wm.boxbackup.org> Author: chris Date: 2012-11-17 00:54:04 +0000 (Sat, 17 Nov 2012) New Revision: 3151 Modified: box/trunk/lib/common/BoxTime.h box/trunk/lib/common/Timer.cpp Log: Fix timer miscalculation on Windows. Modified: box/trunk/lib/common/BoxTime.h =================================================================== --- box/trunk/lib/common/BoxTime.h 2012-11-15 00:00:06 UTC (rev 3150) +++ box/trunk/lib/common/BoxTime.h 2012-11-17 00:54:04 UTC (rev 3151) @@ -18,6 +18,7 @@ #define NANO_SEC_IN_USEC_LL (1000LL) #define MICRO_SEC_IN_SEC (1000000) #define MICRO_SEC_IN_SEC_LL (1000000LL) +#define MICRO_SEC_IN_MILLI_SEC (1000) #define MILLI_SEC_IN_SEC (1000) #define MILLI_SEC_IN_SEC_LL (1000LL) Modified: box/trunk/lib/common/Timer.cpp =================================================================== --- box/trunk/lib/common/Timer.cpp 2012-11-15 00:00:06 UTC (rev 3150) +++ box/trunk/lib/common/Timer.cpp 2012-11-17 00:54:04 UTC (rev 3151) @@ -364,7 +364,7 @@ else { Timers::Add(*this); - Start(timeoutMillis * 1000); + Start(timeoutMillis); } } @@ -393,7 +393,7 @@ timeToExpiry = 1; } - Start(timeToExpiry); + Start(timeToExpiry / MICRO_SEC_IN_MILLI_SEC); #endif } @@ -430,6 +430,10 @@ GetErrorMessage(GetLastError())); mTimerHandle = INVALID_HANDLE_VALUE; } + else + { + BOX_INFO(TIMER_ID << "set for " << timeoutMillis << " ms"); + } #endif } From subversion at boxbackup.org Sat Nov 17 16:54:56 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 17 Nov 2012 16:54:56 GMT Subject: [Box Backup-commit] COMMIT r3152 - box/trunk/bin/bbstoreaccounts Message-ID: <201211171654.qAHGsuMn040334@wm.boxbackup.org> Author: chris Date: 2012-11-17 16:54:55 +0000 (Sat, 17 Nov 2012) New Revision: 3152 Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp Log: Fix account deletion on Windows by removing write lock before deleting files. Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp =================================================================== --- box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp 2012-11-17 00:54:04 UTC (rev 3151) +++ box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp 2012-11-17 16:54:55 UTC (rev 3152) @@ -358,7 +358,7 @@ return 1; } - // Back to original user, but write is maintained + // Back to original user, but write lock is maintained user.reset(); } @@ -403,6 +403,11 @@ } } +#ifdef WIN32 + // Cannot remove files while holding a lock on them + writeLock.ReleaseLock(); +#endif + int retcode = 0; // Thirdly, delete the directories... From subversion at boxbackup.org Sat Nov 17 18:07:29 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 17 Nov 2012 18:07:29 GMT Subject: [Box Backup-commit] COMMIT r3153 - in box/trunk: bin/bbackupd lib/common test/common Message-ID: <201211171807.qAHI7TVn040706@wm.boxbackup.org> Author: chris Date: 2012-11-17 18:07:29 +0000 (Sat, 17 Nov 2012) New Revision: 3153 Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp box/trunk/lib/common/Timer.cpp box/trunk/lib/common/Timer.h box/trunk/test/common/testcommon.cpp Log: Use more efficient direct reset of Timers instead of assignment. Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-11-17 16:54:55 UTC (rev 3152) +++ box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-11-17 18:07:29 UTC (rev 3153) @@ -508,8 +508,7 @@ { mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); - mKeepAliveTimer = Timer(mKeepAliveTime * MILLI_SEC_IN_SEC, - "KeepAliveTime"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); } // -------------------------------------------------------------------------- @@ -569,8 +568,7 @@ BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); mapConnection->QueryGetIsAlive(); - mKeepAliveTimer = Timer(mKeepAliveTime * MILLI_SEC_IN_SEC, - "KeepAliveTime"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); } int BackupClientContext::GetMaximumDiffingTime() Modified: box/trunk/lib/common/Timer.cpp =================================================================== --- box/trunk/lib/common/Timer.cpp 2012-11-17 16:54:55 UTC (rev 3152) +++ box/trunk/lib/common/Timer.cpp 2012-11-17 18:07:29 UTC (rev 3153) @@ -119,6 +119,7 @@ { ASSERT(spTimers); ASSERT(&rTimer); + BOX_TRACE(TIMER_ID_OF(rTimer) " added to global queue, rescheduling"); spTimers->push_back(&rTimer); Reschedule(); } @@ -136,6 +137,7 @@ { ASSERT(spTimers); ASSERT(&rTimer); + BOX_TRACE(TIMER_ID_OF(rTimer) " removed from global queue, rescheduling"); bool restart = true; while (restart) @@ -244,8 +246,8 @@ else { /* - BOX_TRACE("timer " << *i << " has not " - "expired, triggering in " << + BOX_TRACE(TIMER_ID_OF(**i) " has not expired, " + "triggering in " << FORMAT_MICROSECONDS(timeToExpiry) << " seconds"); */ @@ -286,8 +288,8 @@ } else { - BOX_TRACE("timer: next event: " << nameOfNextEvent << - " expires in " << BOX_FORMAT_MICROSECONDS(timeToNextEvent)); + BOX_TRACE("timer: next event: " << nameOfNextEvent << " at " << + FormatTime(timeNow + timeToNextEvent, false, true)); } struct itimerval timeout; @@ -337,52 +339,29 @@ // -------------------------------------------------------------------------- Timer::Timer(size_t timeoutMillis, const std::string& rName) -: mExpires(GetCurrentBoxTime() + MilliSecondsToBoxTime(timeoutMillis)), +: mExpires(0), mExpired(false), mName(rName) #ifdef WIN32 , mTimerHandle(INVALID_HANDLE_VALUE) #endif { - #ifndef BOX_RELEASE_BUILD - if (timeoutMillis == 0) - { - BOX_TRACE(TIMER_ID "initialised for " << timeoutMillis << - " ms, will not fire"); - } - else - { - BOX_TRACE(TIMER_ID "initialised for " << timeoutMillis << - " ms, to fire at " << FormatTime(mExpires, false, true)); - } - #endif - - if (timeoutMillis == 0) - { - mExpires = 0; - } - else - { - Timers::Add(*this); - Start(timeoutMillis); - } + Set(timeoutMillis, true /* isInit */); } // -------------------------------------------------------------------------- // // Function // Name: Timer::Start() -// Purpose: This internal function initialises an OS TimerQueue -// timer on Windows, while on Unixes there is only a -// single global timer, managed by the Timers class, -// so this method does nothing. +// Purpose: This internal function recalculates the remaining +// time (timeout) from the expiry time, and then calls +// Start(timeoutMillis). // Created: 27/07/2008 // // -------------------------------------------------------------------------- void Timer::Start() { -#ifdef WIN32 box_time_t timeNow = GetCurrentBoxTime(); int64_t timeToExpiry = mExpires - timeNow; @@ -394,23 +373,24 @@ } Start(timeToExpiry / MICRO_SEC_IN_MILLI_SEC); -#endif } // -------------------------------------------------------------------------- // // Function // Name: Timer::Start(int64_t timeoutMillis) -// Purpose: This internal function initialises an OS TimerQueue -// timer on Windows, with a specified delay already -// calculated to save us doing it again. Like -// Timer::Start(), on Unixes it does nothing. +// Purpose: This internal function adds this timer to the global +// timer list, and on Windows it initialises an OS +// TimerQueue timer for it. // Created: 27/07/2008 // // -------------------------------------------------------------------------- void Timer::Start(int64_t timeoutMillis) { + ASSERT(mExpires != 0); + Timers::Add(*this); + #ifdef WIN32 // only call me once! ASSERT(mTimerHandle == INVALID_HANDLE_VALUE); @@ -441,15 +421,20 @@ // // Function // Name: Timer::Stop() -// Purpose: This internal function deletes the associated OS -// TimerQueue timer on Windows, and on Unixes does -// nothing. +// Purpose: This internal function removes us from the global +// list of timers, resets our expiry time, and on +// Windows it deletes the associated OS TimerQueue timer. // Created: 27/07/2008 // // -------------------------------------------------------------------------- void Timer::Stop() { + if (mExpires != 0) + { + Timers::Remove(*this); + } + #ifdef WIN32 if (mTimerHandle != INVALID_HANDLE_VALUE) { @@ -479,29 +464,25 @@ BOX_TRACE(TIMER_ID "destroyed"); #endif - Timers::Remove(*this); Stop(); } void Timer::LogAssignment(const Timer &From) { #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "initialised from " << TIMER_ID_OF(From)); + if (From.mExpired) { - BOX_TRACE(TIMER_ID "initialised from timer " << - TIMER_ID_OF(From) << ", already expired, " - "will not fire"); + BOX_TRACE(TIMER_ID "already expired, will not fire"); } else if (From.mExpires == 0) { - BOX_TRACE(TIMER_ID "initialised from timer " << - TIMER_ID_OF(From) << ", no expiry, " - "will not fire"); + BOX_TRACE(TIMER_ID "has no expiry time, will not fire"); } else { - BOX_TRACE(TIMER_ID "initialised from timer " << - TIMER_ID_OF(From) << ", to fire at " << + BOX_TRACE(TIMER_ID "will fire at " << FormatTime(From.mExpires, false, true)); } #endif @@ -530,7 +511,6 @@ if (!mExpired && mExpires != 0) { - Timers::Add(*this); Start(); } } @@ -551,7 +531,6 @@ { LogAssignment(rToCopy); - Timers::Remove(*this); Stop(); mExpires = rToCopy.mExpires; @@ -560,7 +539,6 @@ if (!mExpired && mExpires != 0) { - Timers::Add(*this); Start(); } @@ -570,6 +548,67 @@ // -------------------------------------------------------------------------- // // Function +// Name: Timer::Reset(size_t timeoutMillis) +// Purpose: Simple reset operation for an existing Timer. Avoids +// the need to create a temporary timer just to modify +// an existing one. +// Created: 17/11/2012 +// +// -------------------------------------------------------------------------- + +void Timer::Reset(size_t timeoutMillis) +{ + Set(timeoutMillis, false /* isInit */); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Reset(size_t timeoutMillis) +// Purpose: Internal set/reset operation for an existing Timer. +// Shared by constructor and Reset(). +// Created: 17/11/2012 +// +// -------------------------------------------------------------------------- + +void Timer::Set(size_t timeoutMillis, bool isInit) +{ + Stop(); + mExpired = false; + + if (timeoutMillis == 0) + { + mExpires = 0; + } + else + { + mExpires = GetCurrentBoxTime() + + MilliSecondsToBoxTime(timeoutMillis); + } + + #ifndef BOX_RELEASE_BUILD + if (timeoutMillis == 0) + { + BOX_TRACE(TIMER_ID << (isInit ? "initialised" : "reset") << + " for " << timeoutMillis << " ms, will not fire"); + } + else + { + BOX_TRACE(TIMER_ID << (isInit ? "initialised" : "reset") << + " for " << timeoutMillis << " ms, to fire at " << + FormatTime(mExpires, false, true)); + } + #endif + + if (mExpires != 0) + { + Start(timeoutMillis); + } +} + +// -------------------------------------------------------------------------- +// +// Function // Name: Timer::OnExpire() // Purpose: Method called by Timers::Reschedule (on Unixes) // on next poll after timer expires, or from Modified: box/trunk/lib/common/Timer.h =================================================================== --- box/trunk/lib/common/Timer.h 2012-11-17 16:54:55 UTC (rev 3152) +++ box/trunk/lib/common/Timer.h 2012-11-17 18:07:29 UTC (rev 3153) @@ -67,6 +67,7 @@ } const std::string& GetName() const { return mName; } + virtual void Reset(size_t timeoutMillis); private: box_time_t mExpires; @@ -77,6 +78,7 @@ void Start(int64_t timeoutMillis); void Stop(); void LogAssignment(const Timer &From); + virtual void Set(size_t timeoutMillis, bool isReset); #ifdef WIN32 HANDLE mTimerHandle; Modified: box/trunk/test/common/testcommon.cpp =================================================================== --- box/trunk/test/common/testcommon.cpp 2012-11-17 16:54:55 UTC (rev 3152) +++ box/trunk/test/common/testcommon.cpp 2012-11-17 18:07:29 UTC (rev 3153) @@ -352,12 +352,14 @@ TEST_THAT(t1.HasExpired()); TEST_THAT(t2.HasExpired()); TEST_THAT(!t3.HasExpired()); - + + // Try both ways of resetting an existing timer. t1 = Timer(1000, "t1a"); - t2 = Timer(2000, "t2a"); + t2.Reset(2000); TEST_THAT(!t0.HasExpired()); TEST_THAT(!t1.HasExpired()); TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); safe_sleep(1); TEST_THAT(!t0.HasExpired()); @@ -365,6 +367,12 @@ TEST_THAT(!t2.HasExpired()); TEST_THAT(t3.HasExpired()); + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(t2.HasExpired()); + TEST_THAT(t3.HasExpired()); + // Leave timers initialised for rest of test. // Test main() will cleanup after test finishes. From subversion at boxbackup.org Sat Nov 17 19:49:15 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 17 Nov 2012 19:49:15 GMT Subject: [Box Backup-commit] COMMIT r3154 - box/trunk/lib/server Message-ID: <201211171949.qAHJnFII041198@wm.boxbackup.org> Author: chris Date: 2012-11-17 19:49:14 +0000 (Sat, 17 Nov 2012) New Revision: 3154 Modified: box/trunk/lib/server/ServerStream.h Log: Fix missing space in log message. Modified: box/trunk/lib/server/ServerStream.h =================================================================== --- box/trunk/lib/server/ServerStream.h 2012-11-17 18:07:29 UTC (rev 3153) +++ box/trunk/lib/server/ServerStream.h 2012-11-17 19:49:14 UTC (rev 3154) @@ -297,7 +297,7 @@ // Log it BOX_TRACE("Forked child process " << pid << - "to handle connection from " << + " to handle connection from " << mConnectionDetails); } else From subversion at boxbackup.org Sat Nov 17 19:59:51 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 17 Nov 2012 19:59:51 GMT Subject: [Box Backup-commit] COMMIT r3155 - in box/trunk: bin/bbstored lib/backupstore lib/common Message-ID: <201211171959.qAHJxpBj041261@wm.boxbackup.org> Author: chris Date: 2012-11-17 19:59:51 +0000 (Sat, 17 Nov 2012) New Revision: 3155 Modified: box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp box/trunk/lib/backupstore/HousekeepStoreAccount.cpp box/trunk/lib/backupstore/HousekeepStoreAccount.h box/trunk/lib/common/Logging.h Log: Fix duplicate tagging of housekeeping messages. Modified: box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp =================================================================== --- box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp 2012-11-17 19:49:14 UTC (rev 3154) +++ box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp 2012-11-17 19:59:51 UTC (rev 3155) @@ -100,15 +100,23 @@ { try { - // Tag log output to identify account - std::ostringstream tag; - tag << "hk/" << BOX_FORMAT_ACCOUNT(*i); - Logging::Tagger tagWithClientID(tag.str()); - - // Get the account root std::string rootDir; int discSet = 0; - mpAccounts->GetAccountRoot(*i, rootDir, discSet); + + { + // Tag log output to identify account + std::ostringstream tag; + tag << "hk/" << BOX_FORMAT_ACCOUNT(*i); + Logging::Tagger tagWithClientID(tag.str()); + + // Get the account root + mpAccounts->GetAccountRoot(*i, rootDir, discSet); + + // Reset tagging as HousekeepStoreAccount will + // do that itself, to avoid duplicate tagging. + // Happens automatically when tagWithClientID + // goes out of scope. + } // Do housekeeping on this account HousekeepStoreAccount housekeeping(*i, rootDir, Modified: box/trunk/lib/backupstore/HousekeepStoreAccount.cpp =================================================================== --- box/trunk/lib/backupstore/HousekeepStoreAccount.cpp 2012-11-17 19:49:14 UTC (rev 3154) +++ box/trunk/lib/backupstore/HousekeepStoreAccount.cpp 2012-11-17 19:59:51 UTC (rev 3155) @@ -65,6 +65,9 @@ mRefCountsAdjusted(0), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { + std::ostringstream tag; + tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID); + mTagWithClientID.Change(tag.str()); } // -------------------------------------------------------------------------- @@ -128,7 +131,7 @@ std::ostringstream tag; tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" << info->GetAccountName(); - Logging::Tagger tagWithClientID(tag.str()); + mTagWithClientID.Change(tag.str()); // Calculate how much should be deleted mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit(); Modified: box/trunk/lib/backupstore/HousekeepStoreAccount.h =================================================================== --- box/trunk/lib/backupstore/HousekeepStoreAccount.h 2012-11-17 19:49:14 UTC (rev 3154) +++ box/trunk/lib/backupstore/HousekeepStoreAccount.h 2012-11-17 19:59:51 UTC (rev 3155) @@ -105,6 +105,8 @@ // Poll frequency int mCountUntilNextInterprocessMsgCheck; + + Logging::Tagger mTagWithClientID; }; #endif // HOUSEKEEPSTOREACCOUNT__H Modified: box/trunk/lib/common/Logging.h =================================================================== --- box/trunk/lib/common/Logging.h 2012-11-17 19:49:14 UTC (rev 3154) +++ box/trunk/lib/common/Logging.h 2012-11-17 19:59:51 UTC (rev 3155) @@ -346,15 +346,24 @@ std::string mOldTag; public: + Tagger() + : mOldTag(Logging::GetProgramName()) + { + } Tagger(const std::string& rTempTag) + : mOldTag(Logging::GetProgramName()) { - mOldTag = Logging::GetProgramName(); Logging::SetProgramName(mOldTag + " " + rTempTag); } ~Tagger() { Logging::SetProgramName(mOldTag); } + + void Change(const std::string& newTempTag) + { + Logging::SetProgramName(mOldTag + " " + newTempTag); + } }; }; From subversion at boxbackup.org Sun Nov 18 16:26:43 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 18 Nov 2012 16:26:43 GMT Subject: [Box Backup-commit] COMMIT r3156 - box/trunk/lib/backupstore Message-ID: <201211181626.qAIGQhjx047902@wm.boxbackup.org> Author: chris Date: 2012-11-18 16:26:43 +0000 (Sun, 18 Nov 2012) New Revision: 3156 Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp Log: Allow passing a std::string instead of char* to BackupStoreAccountDatabase, for C++ style. Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp 2012-11-17 19:59:51 UTC (rev 3155) +++ box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp 2012-11-18 16:26:43 UTC (rev 3156) @@ -40,7 +40,7 @@ // Created: 2003/08/20 // // -------------------------------------------------------------------------- -BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename) +BackupStoreAccountDatabase::BackupStoreAccountDatabase(const std::string& Filename) : pImpl(new _BackupStoreAccountDatabase) { pImpl->mFilename = Filename; @@ -123,7 +123,7 @@ // Created: 2003/08/21 // // -------------------------------------------------------------------------- -std::auto_ptr BackupStoreAccountDatabase::Read(const char *Filename) +std::auto_ptr BackupStoreAccountDatabase::Read(const std::string& Filename) { // Database object to use std::auto_ptr db(new BackupStoreAccountDatabase(Filename)); From subversion at boxbackup.org Sun Nov 18 16:29:41 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 18 Nov 2012 16:29:41 GMT Subject: [Box Backup-commit] COMMIT r3157 - in box/trunk: bin/bbstoreaccounts lib/backupstore Message-ID: <201211181629.qAIGTfgq047927@wm.boxbackup.org> Author: chris Date: 2012-11-18 16:29:41 +0000 (Sun, 18 Nov 2012) New Revision: 3157 Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp box/trunk/lib/backupstore/BackupStoreAccounts.cpp box/trunk/lib/backupstore/BackupStoreAccounts.h box/trunk/lib/backupstore/BackupStoreCheck.cpp Log: Add a helper in BackupStoreAccounts to get a write lock on an account. Use it in two places to simplify code (not in BackupStoreContext yet, because that wants to communicate with HK process as well). Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp =================================================================== --- box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp 2012-11-18 16:26:43 UTC (rev 3156) +++ box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp 2012-11-18 16:29:41 UTC (rev 3157) @@ -128,37 +128,8 @@ } } -bool GetWriteLockOnAccount(NamedLock &rLock, const std::string rRootDir, - int discSetNum) -{ - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(rRootDir, discSetNum, writeLockFilename); - - bool gotLock = false; - int triesLeft = 8; - do - { - gotLock = rLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) - { - --triesLeft; - ::sleep(1); - } - } while(!gotLock && triesLeft > 0); - - if(!gotLock) - { - // Couldn't lock the account -- just stop now - BOX_ERROR("Failed to lock the account, did not change limits. " - "Try again later."); - } - - return gotLock; -} - bool OpenAccount(Configuration &rConfig, int32_t ID, std::string &rRootDirOut, - int &rDiscSetOut, std::auto_ptr apUser); + int &rDiscSetOut, std::auto_ptr apUser, NamedLock* pLock); int SetLimit(Configuration &rConfig, int32_t ID, const char *SoftLimitStr, const char *HardLimitStr) @@ -166,22 +137,15 @@ std::string rootDir; int discSetNum; std::auto_ptr user; // used to reset uid when we return + NamedLock writeLock; - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to change limits."); return 1; } - // Attempt to lock - NamedLock writeLock; - if(!GetWriteLockOnAccount(writeLock, rootDir, discSetNum)) - { - // Failed to get lock - return 1; - } - // Load the info std::auto_ptr info(BackupStoreInfo::Load(ID, rootDir, discSetNum, false /* Read/Write */)); @@ -208,22 +172,15 @@ std::string rootDir; int discSetNum; std::auto_ptr user; // used to reset uid when we return + NamedLock writeLock; - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to change name."); return 1; } - // Attempt to lock - NamedLock writeLock; - if(!GetWriteLockOnAccount(writeLock, rootDir, discSetNum)) - { - // Failed to get lock - return 1; - } - // Load the info std::auto_ptr info(BackupStoreInfo::Load(ID, rootDir, discSetNum, false /* Read/Write */)); @@ -245,7 +202,8 @@ int discSetNum; std::auto_ptr user; // used to reset uid when we return - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, + NULL /* no write lock needed for this read-only operation */)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to display info."); @@ -306,8 +264,9 @@ std::string rootDir; int discSetNum; std::auto_ptr user; // used to reset uid when we return + NamedLock writeLock; - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to change enabled flag."); @@ -327,8 +286,10 @@ std::string rootDir; int discSetNum; std::auto_ptr user; // used to reset uid when we return + NamedLock writeLock; - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + // Obtain a write lock, as the daemon user + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " for deletion."); @@ -348,19 +309,8 @@ } } - // Obtain a write lock, as the daemon user - NamedLock writeLock; - { - // Get a write lock - if(!GetWriteLockOnAccount(writeLock, rootDir, discSetNum)) - { - // Failed to get lock - return 1; - } - - // Back to original user, but write lock is maintained - user.reset(); - } + // Back to original user, but write lock is maintained + user.reset(); std::auto_ptr db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); @@ -431,7 +381,7 @@ } bool OpenAccount(Configuration &rConfig, int32_t ID, std::string &rRootDirOut, - int &rDiscSetOut, std::auto_ptr apUser) + int &rDiscSetOut, std::auto_ptr apUser, NamedLock* pLock) { // Load in the account database std::auto_ptr db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); @@ -468,6 +418,11 @@ // in the caller. } + if(pLock) + { + acc.LockAccount(ID, *pLock); + } + return true; } @@ -476,8 +431,9 @@ std::string rootDir; int discSetNum; std::auto_ptr user; // used to reset uid when we return + NamedLock writeLock; - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " for checking."); @@ -530,7 +486,8 @@ int discSetNum; std::auto_ptr user; // used to reset uid when we return - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user)) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, + NULL /* housekeeping locks the account itself */)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " for housekeeping."); Modified: box/trunk/lib/backupstore/BackupStoreAccounts.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccounts.cpp 2012-11-18 16:26:43 UTC (rev 3156) +++ box/trunk/lib/backupstore/BackupStoreAccounts.cpp 2012-11-18 16:29:41 UTC (rev 3157) @@ -11,14 +11,16 @@ #include -#include "BoxPortsAndFiles.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreInfo.h" #include "BackupStoreRefCountDatabase.h" +#include "BoxPortsAndFiles.h" #include "RaidFileWrite.h" -#include "BackupStoreInfo.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreConstants.h" +#include "StoreStructure.h" #include "UnixUser.h" #include "MemLeakFindOn.h" @@ -168,4 +170,34 @@ return mrDatabase.EntryExists(ID); } +void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) +{ + const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); + std::string rootDir = MakeAccountRootDir(ID, en.GetDiscSet()); + int discSet = en.GetDiscSet(); + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(rootDir, discSet, writeLockFilename); + + bool gotLock = false; + int triesLeft = 8; + do + { + gotLock = rNamedLock.TryAndGetLock(writeLockFilename, + 0600 /* restrictive file permissions */); + + if(!gotLock) + { + --triesLeft; + ::sleep(1); + } + } + while (!gotLock && triesLeft > 0); + + if (!gotLock) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotLockStoreAccount, "Failed to get exclusive " + "lock on account " << ID); + } +} Modified: box/trunk/lib/backupstore/BackupStoreAccounts.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccounts.h 2012-11-18 16:26:43 UTC (rev 3156) +++ box/trunk/lib/backupstore/BackupStoreAccounts.h 2012-11-18 16:29:41 UTC (rev 3157) @@ -13,6 +13,7 @@ #include #include "BackupStoreAccountDatabase.h" +#include "NamedLock.h" // -------------------------------------------------------------------------- // @@ -41,6 +42,7 @@ { return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); } + void LockAccount(int32_t ID, NamedLock& rNamedLock); private: static std::string MakeAccountRootDir(int32_t ID, int DiscSet); Modified: box/trunk/lib/backupstore/BackupStoreCheck.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreCheck.cpp 2012-11-18 16:26:43 UTC (rev 3156) +++ box/trunk/lib/backupstore/BackupStoreCheck.cpp 2012-11-18 16:29:41 UTC (rev 3157) @@ -81,41 +81,17 @@ // // Function // Name: BackupStoreCheck::Check() -// Purpose: Perform the check on the given account +// Purpose: Perform the check on the given account. You need to +// hold a lock on the account before calling this! // Created: 21/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { - // Lock the account - { - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + ASSERT(FileExists(writeLockFilename)); - bool gotLock = false; - int triesLeft = 8; - do - { - gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) - { - --triesLeft; - ::sleep(1); - } - } while(!gotLock && triesLeft > 0); - - if(!gotLock) - { - // Couldn't lock the account -- just stop now - if(!mQuiet) - { - BOX_ERROR("Failed to lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server."); - } - THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) - } - } - if(!mQuiet && mFixErrors) { BOX_NOTICE("Will fix errors encountered during checking."); From subversion at boxbackup.org Sun Nov 18 16:30:52 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 18 Nov 2012 16:30:52 GMT Subject: [Box Backup-commit] COMMIT r3158 - box/trunk/lib/backupstore Message-ID: <201211181630.qAIGUqqo047949@wm.boxbackup.org> Author: chris Date: 2012-11-18 16:30:52 +0000 (Sun, 18 Nov 2012) New Revision: 3158 Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.h Log: Finish incomplete checkin [3156]. Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccountDatabase.h 2012-11-18 16:29:41 UTC (rev 3157) +++ box/trunk/lib/backupstore/BackupStoreAccountDatabase.h 2012-11-18 16:30:52 UTC (rev 3158) @@ -31,11 +31,11 @@ friend class _BackupStoreAccountDatabase; // to stop compiler warnings ~BackupStoreAccountDatabase(); private: - BackupStoreAccountDatabase(const char *Filename); + BackupStoreAccountDatabase(const std::string& Filename); BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); public: - static std::auto_ptr Read(const char *Filename); + static std::auto_ptr Read(const std::string& Filename); void Write(); class Entry From subversion at boxbackup.org Sun Nov 18 16:31:24 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 18 Nov 2012 16:31:24 GMT Subject: [Box Backup-commit] COMMIT r3159 - box/trunk/lib/backupstore Message-ID: <201211181631.qAIGVONp047966@wm.boxbackup.org> Author: chris Date: 2012-11-18 16:31:24 +0000 (Sun, 18 Nov 2012) New Revision: 3159 Modified: box/trunk/lib/backupstore/HousekeepStoreAccount.cpp Log: Cosmetic improvements to bbstored log message tagging. Modified: box/trunk/lib/backupstore/HousekeepStoreAccount.cpp =================================================================== --- box/trunk/lib/backupstore/HousekeepStoreAccount.cpp 2012-11-18 16:30:52 UTC (rev 3158) +++ box/trunk/lib/backupstore/HousekeepStoreAccount.cpp 2012-11-18 16:31:24 UTC (rev 3159) @@ -66,7 +66,7 @@ mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { std::ostringstream tag; - tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID); + tag << "hk=" << BOX_FORMAT_ACCOUNT(mAccountID); mTagWithClientID.Change(tag.str()); } @@ -127,11 +127,14 @@ BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, true /* Read Only */)); - // Tag log output to identify account - std::ostringstream tag; - tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" << - info->GetAccountName(); - mTagWithClientID.Change(tag.str()); + // If the account has a name, change the logging tag to include it + if(!(info->GetAccountName().empty())) + { + std::ostringstream tag; + tag << "hk=" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" << + info->GetAccountName(); + mTagWithClientID.Change(tag.str()); + } // Calculate how much should be deleted mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit(); From subversion at boxbackup.org Sun Nov 18 16:32:11 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 18 Nov 2012 16:32:11 GMT Subject: [Box Backup-commit] COMMIT r3160 - box/trunk/test/bbackupd Message-ID: <201211181632.qAIGWB4j047985@wm.boxbackup.org> Author: chris Date: 2012-11-18 16:32:11 +0000 (Sun, 18 Nov 2012) New Revision: 3160 Modified: box/trunk/test/bbackupd/testbbackupd.cpp Log: Fix random failures of bbackupd test by locking the account before making changes to it. Modified: box/trunk/test/bbackupd/testbbackupd.cpp =================================================================== --- box/trunk/test/bbackupd/testbbackupd.cpp 2012-11-18 16:31:24 UTC (rev 3159) +++ box/trunk/test/bbackupd/testbbackupd.cpp 2012-11-18 16:32:11 UTC (rev 3160) @@ -49,11 +49,13 @@ #include "BackupDaemon.h" #include "BackupDaemonConfigVerify.h" #include "BackupQueries.h" +#include "BackupStoreAccounts.h" #include "BackupStoreConstants.h" #include "BackupStoreContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" +#include "BackupStoreConfigVerify.h" #include "BoxPortsAndFiles.h" #include "BoxTime.h" #include "BoxTimeToUnix.h" @@ -65,6 +67,7 @@ #include "intercept.h" #include "IOStreamGetLine.h" #include "LocalProcessStream.h" +#include "RaidFileController.h" #include "SSLLib.h" #include "ServerControl.h" #include "Socket.h" @@ -1023,11 +1026,8 @@ Timers::Init(); - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + // stop early to make debugging easier + if (failures) return 1; // four-second delay on first read() of f1 // should mean that no keepalives were sent, @@ -1293,6 +1293,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if(bbackupd_pid > 0) { @@ -1396,11 +1397,7 @@ sSocket.Close(); } - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + if (failures) return 1; // ensure time is different to refresh the cache ::safe_sleep(1); @@ -1535,11 +1532,7 @@ } BOX_TRACE("done."); - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + if (failures) return 1; wait_for_operation(5, "housekeeping to remove the " "deleted files"); @@ -1587,11 +1580,7 @@ sSocket.Close(); } - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + if (failures) return 1; // Need 22 blocks free to upload everything TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " @@ -1629,11 +1618,7 @@ sSocket.Close(); } - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + if (failures) return 1; // Put the limit back TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " @@ -1653,6 +1638,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; BOX_TRACE("done."); // unpack the initial files again @@ -1677,12 +1663,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; - - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + if (failures) return 1; } // Check that no read error has been reported yet @@ -1816,6 +1797,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; } #endif // !WIN32 @@ -1945,6 +1927,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; } #endif // !WIN32 @@ -1974,6 +1957,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; TEST_THAT(!TestFileExists("testfiles/notifyran.backup-start.1")); TEST_THAT(!TestFileExists("testfiles/notifyran.backup-start.2")); @@ -2066,6 +2050,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Test2 should be deleted after 10 seconds (4 runs) wait_for_sync_end(); @@ -2111,6 +2096,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if(bbackupd_pid > 0) { @@ -2510,6 +2496,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Check that SyncAllowScript is executed and can " "pause backup\n"); @@ -2603,11 +2590,7 @@ // check that backup has run (compare succeeds) TEST_COMPARE(Compare_Same); - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } + if (failures) return 1; } // Check that no read error has been reported yet @@ -2617,6 +2600,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Delete file and update another, " "create symlink.\n"); @@ -2668,20 +2652,46 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Check that store errors are reported neatly printf("\n==== Create store error\n"); TEST_THAT(system("rm -f testfiles/notifyran.backup-error.*") == 0); - // break the store - TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", - "testfiles/0_0/backup/01234567/info.rf.bak") == 0); - TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf", - "testfiles/0_1/backup/01234567/info.rf.bak") == 0); - TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf", - "testfiles/0_2/backup/01234567/info.rf.bak") == 0); + // Break the store. We need a write lock on the account + // while we do this, otherwise housekeeping might be running + // and might rewrite the info files when it finishes, + // undoing our breakage. + std::string errs; + std::auto_ptr config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + TEST_EQUAL_LINE(0, errs.size(), "Loading configuration file " + "reported errors: " << errs); + TEST_THAT(config.get() != 0); + // Initialise the raid file controller + RaidFileController &rcontroller(RaidFileController::GetController()); + rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); + std::auto_ptr db( + BackupStoreAccountDatabase::Read( + config->GetKeyValue("AccountDatabase"))); + BackupStoreAccounts acc(*db); + + // Lock scope + { + NamedLock writeLock; + acc.LockAccount(0x01234567, writeLock); + + TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", + "testfiles/0_0/backup/01234567/info.rf.bak") == 0); + TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf", + "testfiles/0_1/backup/01234567/info.rf.bak") == 0); + TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf", + "testfiles/0_2/backup/01234567/info.rf.bak") == 0); + } + // Create a file to trigger an upload { int fd1 = open("testfiles/TestDir1/force-upload", @@ -2709,6 +2719,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; sync_and_wait(); @@ -2795,6 +2806,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; sync_and_wait(); @@ -2816,7 +2828,7 @@ TEST_THAT(!TestFileExists("testfiles/" "notifyran.backup-start.wait-automatic.1")); - // Set a tag for the notify script to distinguist from + // Set a tag for the notify script to distinguish from // previous runs. { int fd1 = open("testfiles/notifyscript.tag", @@ -2851,6 +2863,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Bad case: delete a file/symlink, replace it with a directory printf("\n==== Replace symlink with directory, " @@ -2882,6 +2895,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // And the inverse, replace a directory with a file/symlink printf("\n==== Replace directory with symlink\n"); @@ -2906,6 +2920,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // And then, put it back to how it was before. printf("\n==== Replace symlink with directory " @@ -2933,6 +2948,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // And finally, put it back to how it was before // it was put back to how it was before @@ -2960,6 +2976,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // rename an untracked file over an // existing untracked file @@ -3004,6 +3021,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // case which went wrong: rename a tracked file over an // existing tracked file @@ -3053,6 +3071,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // case which went wrong: rename a tracked file // over a deleted file @@ -3072,6 +3091,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Add files with old times, update " "attributes of one to latest time\n"); @@ -3102,6 +3122,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); @@ -3158,6 +3179,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); @@ -3193,6 +3215,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // check that the excluded files did not make it // into the store, and the included files did @@ -3236,6 +3259,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; #ifndef WIN32 // These tests only work as non-root users. @@ -3284,6 +3308,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Continuously update file, " "check isn't uploaded\n"); @@ -3349,6 +3374,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Delete directory, change attributes\n"); @@ -3446,6 +3472,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; #ifdef WIN32 // make one of the files read-only, expect a compare failure @@ -3503,6 +3530,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Add files with current time\n"); @@ -3525,6 +3553,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Rename directory printf("\n==== Rename directory\n"); @@ -3561,6 +3590,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; // Check that modifying files with madly in the future // timestamps still get added @@ -3598,6 +3628,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Change client store marker\n"); @@ -3640,6 +3671,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Check change of store marker pauses daemon\n"); @@ -3663,14 +3695,18 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; - // 100 seconds - (12*3/2) - wait_for_operation(82, "bbackupd to recover"); + wait_for_operation(100, "bbackupd to recover"); + // Then check it has backed up successfully. + TEST_COMPARE(Compare_Same); + TEST_THAT(ServerIsAlive(bbackupd_pid)); TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; #ifndef WIN32 printf("\n==== Interrupted restore\n"); @@ -3714,14 +3750,7 @@ sSocket.Close(); // Then check it has restored the correct stuff - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-l testfiles/query14.log " - "-Wwarning \"compare -cEQ Test1 " - "testfiles/restore-interrupt\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_COMPARE(Compare_Same); } #endif // !WIN32 @@ -3729,6 +3758,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; printf("\n==== Check restore deleted files\n"); @@ -3769,6 +3799,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; #ifdef WIN32 printf("\n==== Testing locked file behaviour:\n"); @@ -3795,6 +3826,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if (handle != 0) { @@ -3811,6 +3843,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if (handle != 0) { @@ -3828,6 +3861,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if (handle != 0) { @@ -3840,6 +3874,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if (handle != 0) { @@ -3860,6 +3895,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if (handle != 0) { @@ -3881,6 +3917,7 @@ TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if (failures) return 1; if(bbackupd_pid != -1 && bbackupd_pid != 0) { From subversion at boxbackup.org Sun Nov 18 16:34:10 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 18 Nov 2012 16:34:10 GMT Subject: [Box Backup-commit] COMMIT r3161 - box/trunk/test/backupstore Message-ID: <201211181634.qAIGYApq048018@wm.boxbackup.org> Author: chris Date: 2012-11-18 16:34:10 +0000 (Sun, 18 Nov 2012) New Revision: 3161 Modified: box/trunk/test/backupstore/testbackupstore.cpp Log: Fix testbackupstore: don't keep using a reference to a protocol object that's been disposed (Windows only). Modified: box/trunk/test/backupstore/testbackupstore.cpp =================================================================== --- box/trunk/test/backupstore/testbackupstore.cpp 2012-11-18 16:32:11 UTC (rev 3160) +++ box/trunk/test/backupstore/testbackupstore.cpp 2012-11-18 16:34:10 UTC (rev 3161) @@ -1040,7 +1040,7 @@ TEST_THAT(loginConf->GetClientStoreMarker() == 0x8732523ab23aLL); } #else // WIN32 - BackupProtocolClient& protocolReadOnly(*apProtocol); + #define protocolReadOnly (*apProtocol) #endif test_server_1(*apProtocol, protocolReadOnly); @@ -2490,6 +2490,7 @@ *apInfo, "spurtle", false /* AccountEnabled */, extra_data); // Delete the account to leave the store in the same state as before + apInfo.reset(); TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf delete 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");