From boxbackup-dev at boxbackup.org Mon Jun 22 06:33:00 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Mon, 22 Jun 2009 06:33:00 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2528 - box/trunk/bin/bbackupd Message-ID: <20090622053300.165EC325037@www.boxbackup.org> Author: chris Date: 2009-06-22 06:32:59 +0100 (Mon, 22 Jun 2009) New Revision: 2528 Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp Log: Ignore stat() errors on excluded directories. Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2009-05-22 20:01:51 UTC (rev 2527) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2009-06-22 05:32:59 UTC (rev 2528) @@ -294,14 +294,21 @@ #else if(EMU_LSTAT(filename.c_str(), &file_st) != 0) { - // Report the error (logs and - // eventual email to administrator) - rNotifier.NotifyFileStatFailed(this, - filename, strerror(errno)); + if(!(rParams.mrContext.ExcludeDir( + filename))) + { + // Report the error (logs and + // eventual email to + // administrator) + rNotifier.NotifyFileStatFailed( + this, filename, + strerror(errno)); - // FIXME move to NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + // FIXME move to + // NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject( + rParams, filename.c_str()); + } // Ignore this entry for now. continue; From boxbackup-dev at boxbackup.org Sat Jun 27 12:28:16 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sat, 27 Jun 2009 12:28:16 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2529 - box/trunk/lib/raidfile Message-ID: <20090627112816.68970325020@www.boxbackup.org> Author: chris Date: 2009-06-27 12:28:15 +0100 (Sat, 27 Jun 2009) New Revision: 2529 Modified: box/trunk/lib/raidfile/RaidFileWrite.cpp Log: Improve RaidFileWrite error logging. Modified: box/trunk/lib/raidfile/RaidFileWrite.cpp =================================================================== --- box/trunk/lib/raidfile/RaidFileWrite.cpp 2009-06-22 05:32:59 UTC (rev 2528) +++ box/trunk/lib/raidfile/RaidFileWrite.cpp 2009-06-27 11:28:15 UTC (rev 2529) @@ -113,6 +113,7 @@ S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if(mOSFileHandle == -1) { + BOX_LOG_SYS_ERROR("Failed to open file: " << writeFilename); THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile) } @@ -274,13 +275,15 @@ if(::unlink(renameTo.c_str()) != 0 && GetLastError() != ERROR_FILE_NOT_FOUND) { - BOX_LOG_WIN_ERROR("failed to delete file: " << renameTo); + BOX_LOG_WIN_ERROR("Failed to delete file: " << renameTo); THROW_EXCEPTION(RaidFileException, OSError) } #endif if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0) { + BOX_LOG_SYS_ERROR("Failed to rename file: " << renameFrom << + " to " << renameTo); THROW_EXCEPTION(RaidFileException, OSError) } @@ -335,6 +338,7 @@ ::close(mOSFileHandle) != 0) #endif // !WIN32 { + BOX_LOG_SYS_ERROR("Failed to delete file: " << writeFilename); THROW_EXCEPTION(RaidFileException, OSError) } @@ -562,6 +566,8 @@ ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0); if(::write(parity, &sw, sizeof(sw)) != sizeof(sw)) { + BOX_LOG_SYS_ERROR("Failed to write to file: " << + writeFilename); THROW_EXCEPTION(RaidFileException, OSError) } } @@ -600,6 +606,8 @@ // Finally delete the write file if(::unlink(writeFilename.c_str()) != 0) { + BOX_LOG_SYS_ERROR("Failed to delete file: " << + writeFilename); THROW_EXCEPTION(RaidFileException, OSError) } } From boxbackup-dev at boxbackup.org Sat Jun 27 12:34:20 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sat, 27 Jun 2009 12:34:20 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2530 - box/trunk/lib/backupstore Message-ID: <20090627113420.C61C1325020@www.boxbackup.org> Author: chris Date: 2009-06-27 12:34:19 +0100 (Sat, 27 Jun 2009) New Revision: 2530 Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp box/trunk/lib/backupstore/BackupStoreAccountDatabase.h Log: Make BackupStoreAccountDatabase::AddEntry and BackupStoreAccountDatabase::GetEntry return a copy of the new entry. Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp 2009-06-27 11:28:15 UTC (rev 2529) +++ box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp 2009-06-27 11:34:19 UTC (rev 2530) @@ -289,7 +289,8 @@ // Created: 2003/08/21 // // -------------------------------------------------------------------------- -const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const +BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::GetEntry( + int32_t ID) const { // Check that we're using the latest version of the database CheckUpToDate(); @@ -311,12 +312,14 @@ // Created: 2003/08/21 // // -------------------------------------------------------------------------- -void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet) +BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::AddEntry( + int32_t ID, int DiscSet) { // Check that we're using the latest version of the database CheckUpToDate(); pImpl->mDatabase[ID] = Entry(ID, DiscSet); + return pImpl->mDatabase[ID]; } Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccountDatabase.h 2009-06-27 11:28:15 UTC (rev 2529) +++ box/trunk/lib/backupstore/BackupStoreAccountDatabase.h 2009-06-27 11:34:19 UTC (rev 2530) @@ -55,8 +55,8 @@ }; bool EntryExists(int32_t ID) const; - const Entry &GetEntry(int32_t ID) const; - void AddEntry(int32_t ID, int DiscSet); + Entry GetEntry(int32_t ID) const; + Entry AddEntry(int32_t ID, int DiscSet); void DeleteEntry(int32_t ID); // This interface should change in the future. But for now it'll do. From boxbackup-dev at boxbackup.org Sat Jun 27 12:34:41 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sat, 27 Jun 2009 12:34:41 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2531 - box/trunk/lib/backupclient Message-ID: <20090627113441.D8567325020@www.boxbackup.org> Author: chris Date: 2009-06-27 12:34:41 +0100 (Sat, 27 Jun 2009) New Revision: 2531 Modified: box/trunk/lib/backupclient/BackupStoreException.txt Log: Add a new exception type for refcount database errors. Modified: box/trunk/lib/backupclient/BackupStoreException.txt =================================================================== --- box/trunk/lib/backupclient/BackupStoreException.txt 2009-06-27 11:34:19 UTC (rev 2530) +++ box/trunk/lib/backupclient/BackupStoreException.txt 2009-06-27 11:34:41 UTC (rev 2531) @@ -68,3 +68,4 @@ IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. +UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. From boxbackup-dev at boxbackup.org Sat Jun 27 12:35:02 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sat, 27 Jun 2009 12:35:02 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2532 - box/trunk/lib/common Message-ID: <20090627113502.ACD65326026@www.boxbackup.org> Author: chris Date: 2009-06-27 12:35:02 +0100 (Sat, 27 Jun 2009) New Revision: 2532 Modified: box/trunk/lib/common/Utils.cpp Log: Make FileExists take a std::string instead of a const char * for C++ style. Modified: box/trunk/lib/common/Utils.cpp =================================================================== --- box/trunk/lib/common/Utils.cpp 2009-06-27 11:34:41 UTC (rev 2531) +++ box/trunk/lib/common/Utils.cpp 2009-06-27 11:35:02 UTC (rev 2532) @@ -156,15 +156,16 @@ // -------------------------------------------------------------------------- // // Function -// Name: FileExists(const char *) +// Name: FileExists(const std::string& rFilename) // Purpose: Does a file exist? // Created: 20/11/03 // // -------------------------------------------------------------------------- -bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotExisting) +bool FileExists(const std::string& rFilename, int64_t *pFileSize, + bool TreatLinksAsNotExisting) { EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) != 0) + if(EMU_LSTAT(rFilename.c_str(), &st) != 0) { if(errno == ENOENT) { From boxbackup-dev at boxbackup.org Sat Jun 27 12:35:15 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sat, 27 Jun 2009 12:35:15 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2533 - box/trunk/lib/common Message-ID: <20090627113515.ACC32325020@www.boxbackup.org> Author: chris Date: 2009-06-27 12:35:15 +0100 (Sat, 27 Jun 2009) New Revision: 2533 Modified: box/trunk/lib/common/Utils.h Log: Second part of FileExists change. Modified: box/trunk/lib/common/Utils.h =================================================================== --- box/trunk/lib/common/Utils.h 2009-06-27 11:35:02 UTC (rev 2532) +++ box/trunk/lib/common/Utils.h 2009-06-27 11:35:15 UTC (rev 2533) @@ -21,7 +21,8 @@ void DumpStackBacktrace(); #endif -bool FileExists(const char *Filename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false); +bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0, + bool TreatLinksAsNotExisting = false); enum { From boxbackup-dev at boxbackup.org Sat Jun 27 12:38:53 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sat, 27 Jun 2009 12:38:53 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2534 - box/trunk/lib/backupstore Message-ID: <20090627113853.12D29325020@www.boxbackup.org> Author: chris Date: 2009-06-27 12:38:52 +0100 (Sat, 27 Jun 2009) New Revision: 2534 Added: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h Modified: box/trunk/lib/backupstore/BackupStoreAccounts.cpp box/trunk/lib/backupstore/BackupStoreAccounts.h Log: Add code for BackupStoreRefCountDatabase. Create a refcount database when creating a new account. Add an easier way to get the account root directory. Modified: box/trunk/lib/backupstore/BackupStoreAccounts.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccounts.cpp 2009-06-27 11:35:15 UTC (rev 2533) +++ box/trunk/lib/backupstore/BackupStoreAccounts.cpp 2009-06-27 11:38:52 UTC (rev 2534) @@ -14,6 +14,7 @@ #include "BoxPortsAndFiles.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" #include "RaidFileWrite.h" #include "BackupStoreInfo.h" #include "BackupStoreDirectory.h" @@ -61,6 +62,10 @@ // -------------------------------------------------------------------------- void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) { + // Create the entry in the database + BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID, + DiscSet)); + { // Become the user specified in the config file? std::auto_ptr user; @@ -100,15 +105,17 @@ // Save it back info->Save(); + + // Create the refcount database + BackupStoreRefCountDatabase::CreateNew(Entry); + std::auto_ptr refcount( + BackupStoreRefCountDatabase::Load(Entry, false)); + refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); } // As the original user... - - // Create the entry in the database - mrDatabase.AddEntry(ID, DiscSet); - // Write the database back - mrDatabase.Write(); + mrDatabase.Write(); } @@ -138,7 +145,7 @@ // Created: 2003/08/21 // // -------------------------------------------------------------------------- -std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const +std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) { char accid[64]; // big enough! ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); Modified: box/trunk/lib/backupstore/BackupStoreAccounts.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccounts.h 2009-06-27 11:35:15 UTC (rev 2533) +++ box/trunk/lib/backupstore/BackupStoreAccounts.h 2009-06-27 11:38:52 UTC (rev 2534) @@ -12,7 +12,7 @@ #include -class BackupStoreAccountDatabase; +#include "BackupStoreAccountDatabase.h" // -------------------------------------------------------------------------- // @@ -35,9 +35,14 @@ bool AccountExists(int32_t ID); void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; + static std::string GetAccountRoot(const + BackupStoreAccountDatabase::Entry &rEntry) + { + return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); + } private: - std::string MakeAccountRootDir(int32_t ID, int DiscSet) const; + static std::string MakeAccountRootDir(int32_t ID, int DiscSet); private: BackupStoreAccountDatabase &mrDatabase; Copied: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp (from rev 2527, box/trunk/lib/backupstore/BackupStoreInfo.cpp) =================================================================== --- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp (rev 0) +++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp 2009-06-27 11:38:52 UTC (rev 2534) @@ -0,0 +1,319 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.cpp +// Purpose: Backup store object reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BackupStoreRefCountDatabase.h" +#include "BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreAccounts.h" +#include "RaidFileController.h" +#include "RaidFileUtil.h" +#include "RaidFileException.h" + +#include "MemLeakFindOn.h" + +#define REFCOUNT_MAGIC_VALUE 0x52656643 // RefC +#define REFCOUNT_FILENAME "refcount" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::BackupStoreRefCountDatabase() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const + BackupStoreAccountDatabase::Entry& rAccount) +: mAccount(rAccount), + mFilename(GetFilename(rAccount)), + mReadOnly(true), + mIsModified(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() +{ +} + +std::string BackupStoreRefCountDatabase::GetFilename(const + BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + ASSERT(RootDir[RootDir.size() - 1] == '/' || + RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + + std::string fn(RootDir + "refcount.db"); + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); + return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Create(int32_t, +// const std::string &, int, bool) +// Purpose: Create a new database, overwriting an existing +// one only if AllowOverwrite is true. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreRefCountDatabase::Create(const + BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +{ + // Initial header + refcount_StreamFormat hdr; + hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); + hdr.mAccountID = htonl(rAccount.GetID()); + + // Generate the filename + std::string Filename = GetFilename(rAccount); + + // Open the file for writing + if (FileExists(Filename) && !AllowOverwrite) + { + BOX_ERROR("Attempted to overwrite refcount database file: " << + Filename); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile); + } + + int flags = O_CREAT | O_BINARY | O_RDWR; + if (!AllowOverwrite) + { + flags |= O_EXCL; + } + + std::auto_ptr DatabaseFile(new FileStream(Filename, flags)); + + // Write header + DatabaseFile->Write(&hdr, sizeof(hdr)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Load(int32_t AccountID, +// BackupStoreAccountDatabase& rAccountDatabase, +// bool ReadOnly); +// Purpose: Loads the info from disc, given the root +// information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreRefCountDatabase::Load( + const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) +{ + // Generate the filename + std::string filename = GetFilename(rAccount); + int flags = ReadOnly ? O_RDONLY : O_RDWR; + + // Open the file for read/write + std::auto_ptr dbfile(new FileStream(filename, + flags | O_BINARY)); + + // Read in a header + refcount_StreamFormat hdr; + if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || + (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr refcount(new BackupStoreRefCountDatabase(rAccount)); + + // Put in basic location info + refcount->mReadOnly = ReadOnly; + refcount->mapDatabaseFile = dbfile; + + // return it to caller + return refcount; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +/* +void BackupStoreRefCountDatabase::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccount.GetID() == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = box_hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} +*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::GetRefCount(int64_t +// ObjectID) +// Purpose: Get the number of references to the specified object +// out of the database +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const +{ + IOStream::pos_type offset = GetOffset(ObjectID); + + if (GetSize() < offset + GetEntrySize()) + { + BOX_ERROR("attempted read of unknown refcount for object " << + BOX_FORMAT_OBJECTID(ObjectID)); + THROW_EXCEPTION(BackupStoreException, + UnknownObjectRefCountRequested); + } + + mapDatabaseFile->Seek(offset, SEEK_SET); + + refcount_t refcount; + if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != + sizeof(refcount)) + { + BOX_LOG_SYS_ERROR("short read on refcount database: " << + mFilename); + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo); + } + + return ntohl(refcount); +} + +int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const +{ + return (GetSize() - sizeof(refcount_StreamFormat)) / + sizeof(refcount_t); +} + +void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID) +{ + refcount_t refcount; + + if (ObjectID > GetLastObjectIDUsed()) + { + // new object, assume no previous references + refcount = 0; + } + else + { + // read previous value from database + refcount = GetRefCount(ObjectID); + } + + refcount++; + + SetRefCount(ObjectID, refcount); +} + +void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, + refcount_t NewRefCount) +{ + IOStream::pos_type offset = GetOffset(ObjectID); + mapDatabaseFile->Seek(offset, SEEK_SET); + refcount_t RefCountNetOrder = htonl(NewRefCount); + mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); +} + +bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) +{ + refcount_t refcount = GetRefCount(ObjectID); // must exist in database + ASSERT(refcount > 0); + refcount--; + SetRefCount(ObjectID, refcount); + return (refcount > 0); +} + Copied: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h (from rev 2527, box/trunk/lib/backupstore/BackupStoreInfo.h) =================================================================== --- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h (rev 0) +++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h 2009-06-27 11:38:52 UTC (rev 2534) @@ -0,0 +1,125 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREREFCOUNTDATABASE__H +#define BACKUPSTOREREFCOUNTDATABASE__H + +#include +#include +#include + +#include "BackupStoreAccountDatabase.h" +#include "FileStream.h" + +class BackupStoreCheck; +class BackupStoreContext; + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + uint32_t mMagicValue; // also the version number + uint32_t mAccountID; +} refcount_StreamFormat; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreRefCountDatabase +// Purpose: Backup store reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +class BackupStoreRefCountDatabase +{ + friend class BackupStoreCheck; + friend class BackupStoreContext; +public: + ~BackupStoreRefCountDatabase(); +private: + // Creation through static functions only + BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& + rAccount); + // No copying allowed + BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); + +public: + // Create a new database for a new account. This method will refuse + // to overwrite any existing file. + static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, false); + } + + // Load it from the store + static std::auto_ptr Load(const + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + + // Data access functions + int32_t GetRefCount(int64_t ObjectID) const; + int64_t GetLastObjectIDUsed() const; + + // Data modification functions + void AddReference(int64_t ObjectID); + bool RemoveReference(int64_t ObjectID); + +private: + // Create a new database for an existing account. Used during + // account checking if opening the old database throws an exception. + // This method will overwrite any existing file. + static void CreateForRegeneration(const + BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, true); + } + + static void Create(const BackupStoreAccountDatabase::Entry& rAccount, + bool AllowOverwrite); + + typedef int32_t refcount_t; + + static std::string GetFilename(const BackupStoreAccountDatabase::Entry& + rAccount); + IOStream::pos_type GetSize() const + { + return mapDatabaseFile->GetPosition() + + mapDatabaseFile->BytesLeftToRead(); + } + IOStream::pos_type GetEntrySize() const + { + return sizeof(refcount_t); + } + IOStream::pos_type GetOffset(int64_t ObjectID) const + { + return ((ObjectID - 1) * GetEntrySize()) + + sizeof(refcount_StreamFormat); + } + void SetRefCount(int64_t ObjectID, refcount_t NewRefCount); + + // Location information + BackupStoreAccountDatabase::Entry mAccount; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + std::auto_ptr mapDatabaseFile; +}; + +#endif // BACKUPSTOREREFCOUNTDATABASE__H From boxbackup-dev at boxbackup.org Sun Jun 28 13:04:26 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 13:04:26 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2535 - in box/trunk: bin/bbstored test/backupstore Message-ID: <20090628120426.429F1325020@www.boxbackup.org> Author: chris Date: 2009-06-28 13:04:25 +0100 (Sun, 28 Jun 2009) New Revision: 2535 Modified: box/trunk/bin/bbstored/BackupStoreContext.cpp box/trunk/bin/bbstored/BackupStoreContext.h box/trunk/test/backupstore/testbackupstore.cpp Log: Update reference counts when files and directories are uploaded. Modified: box/trunk/bin/bbstored/BackupStoreContext.cpp =================================================================== --- box/trunk/bin/bbstored/BackupStoreContext.cpp 2009-06-27 11:38:52 UTC (rev 2534) +++ box/trunk/bin/bbstored/BackupStoreContext.cpp 2009-06-28 12:04:25 UTC (rev 2535) @@ -191,6 +191,23 @@ // Keep the pointer to it mpStoreInfo = i; + + BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet); + + // try to load the reference count database + try + { + mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database is missing or corrupted, " + "creating a new one, expect housekeeping to find and " + "fix problems with reference counts later."); + + BackupStoreRefCountDatabase::CreateForRegeneration(account); + mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + } } @@ -395,15 +412,18 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) -// Purpose: Add a file to the store, from a given stream, into a specified directory. -// Returns object ID of the new file. +// Name: BackupStoreContext::AddFile(IOStream &, int64_t, +// int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Add a file to the store, from a given stream, into +// a specified directory. Returns object ID of the new +// file. // Created: 2003/09/03 // // -------------------------------------------------------------------------- -int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, - int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, - bool MarkFileWithSameNameAsOldVersions) +int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, + int64_t ModificationTime, int64_t AttributesHash, + int64_t DiffFromFileID, const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions) { if(mpStoreInfo.get() == 0) { @@ -670,6 +690,9 @@ mpStoreInfo->ChangeBlocksUsed(blocksUsed); mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); + // Increment reference count on the new directory to one + mapRefCount->AddReference(id); + // Save the store info -- can cope if this exceptions because infomation // will be rebuilt by housekeeping, and ID allocation can recover. SaveStoreInfo(); @@ -768,8 +791,9 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t) +// Purpose: Undeletes a file, if it exists, returning true if +// the file existed. // Created: 2003/10/21 // // -------------------------------------------------------------------------- @@ -933,8 +957,10 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &) -// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely. +// Name: BackupStoreContext::AddDirectory(int64_t, +// const BackupStoreFilename &, bool &) +// Purpose: Creates a directory (or just returns the ID of an +// existing one). rAlreadyExists set appropraitely. // Created: 2003/09/04 // // -------------------------------------------------------------------------- @@ -974,7 +1000,7 @@ // Allocate the next ID int64_t id = AllocateObjectID(); - // Create a blank directory with the given attributes on disc + // Create an empty directory with the given attributes on disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); { @@ -998,11 +1024,14 @@ // Not added to cache, so don't set the size in the directory } - // Then add it into the directory + // Then add it into the parent directory try { dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); SaveDirectory(dir, InDirectory); + + // Increment reference count on the new directory to one + mapRefCount->AddReference(id); } catch(...) { @@ -1016,7 +1045,7 @@ // Don't worry about the incremented number in the store info throw; } - + // Save the store info (may be postponed) SaveStoreInfo(); Modified: box/trunk/bin/bbstored/BackupStoreContext.h =================================================================== --- box/trunk/bin/bbstored/BackupStoreContext.h 2009-06-27 11:38:52 UTC (rev 2534) +++ box/trunk/bin/bbstored/BackupStoreContext.h 2009-06-28 12:04:25 UTC (rev 2535) @@ -14,6 +14,7 @@ #include #include +#include "BackupStoreRefCountDatabase.h" #include "NamedLock.h" #include "ProtocolObject.h" #include "Utils.h" @@ -137,7 +138,6 @@ void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); int64_t AllocateObjectID(); -private: int32_t mClientID; HousekeepingInterface &mrDaemon; int mProtocolPhase; @@ -150,6 +150,9 @@ // Store info std::auto_ptr mpStoreInfo; + + // Refcount database + std::auto_ptr mapRefCount; // Directory cache std::map mDirectoryCache; Modified: box/trunk/test/backupstore/testbackupstore.cpp =================================================================== --- box/trunk/test/backupstore/testbackupstore.cpp 2009-06-27 11:38:52 UTC (rev 2534) +++ box/trunk/test/backupstore/testbackupstore.cpp 2009-06-28 12:04:25 UTC (rev 2535) @@ -34,6 +34,8 @@ #include "BackupClientFileAttributes.h" #include "BackupClientCryptoKeys.h" #include "ServerControl.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" #include "MemLeakFindOn.h" @@ -470,8 +472,28 @@ TEST_THAT(dirs == 0 || dirs == 2); } -int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth) +void create_file_in_dir(std::string name, std::string source, int64_t parentId, + BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount) { + BackupStoreFilenameClear name_encoded("file_One"); + std::auto_ptr upload(BackupStoreFile::EncodeFile( + source.c_str(), parentId, name_encoded)); + std::auto_ptr stored( + protocol.QueryStoreFile( + parentId, + 0x123456789abcdefLL, /* modification time */ + 0x7362383249872dfLL, /* attr hash */ + 0, /* diff from ID */ + name_encoded, + *upload)); + int64_t objectId = stored->GetObjectID(); + TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed()); + TEST_EQUAL(1, rRefCount.GetRefCount(objectId)) +} + +int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, + const char *name, int depth, BackupStoreRefCountDatabase& rRefCount) +{ // Create a directory int64_t subdirid = 0; BackupStoreFilenameClear dirname(name); @@ -486,49 +508,31 @@ } printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid); + + std::auto_ptr apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr apReferences( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); + TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); + TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) // Put more directories in it, if we haven't gone down too far if(depth > 0) { - create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1); - create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1); + create_test_data_subdirs(protocol, subdirid, "dir_One", + depth - 1, rRefCount); + create_test_data_subdirs(protocol, subdirid, "dir_Two", + depth - 1, rRefCount); } // Stick some files in it - { - BackupStoreFilenameClear name("file_One"); - std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name)); - std::auto_ptr stored(protocol.QueryStoreFile( - subdirid, - 0x123456789abcdefLL, /* modification time */ - 0x7362383249872dfLL, /* attr hash */ - 0, /* diff from ID */ - name, - *upload)); - } - { - BackupStoreFilenameClear name("file_Two"); - std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name)); - std::auto_ptr stored(protocol.QueryStoreFile( - subdirid, - 0x123456789abcdefLL, /* modification time */ - 0x7362383249872dfLL, /* attr hash */ - 0, /* diff from ID */ - name, - *upload)); - } - { - BackupStoreFilenameClear name("file_Three"); - std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name)); - std::auto_ptr stored(protocol.QueryStoreFile( - subdirid, - 0x123456789abcdefLL, /* modification time */ - 0x7362383249872dfLL, /* attr hash */ - 0, /* diff from ID */ - name, - *upload)); - } - + create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol, + rRefCount); + create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol, + rRefCount); + create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol, + rRefCount); return subdirid; } @@ -885,16 +889,49 @@ } } - -int test_server(const char *hostname) +void init_context(TLSContext& rContext) { - // Context - TLSContext context; - context.Initialise(false /* client */, + rContext.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); +} +std::auto_ptr open_conn(const char *hostname, + TLSContext& rContext) +{ + init_context(rContext); + std::auto_ptr conn(new SocketStreamTLS); + conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); + return conn; +} + +std::auto_ptr test_server_login(SocketStreamTLS& rConn) +{ + // Make a protocol + std::auto_ptr protocol(new + BackupProtocolClient(rConn)); + + // Check the version + std::auto_ptr serverVersion( + protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + + // Login + std::auto_ptr loginConf( + protocol->QueryLogin(0x01234567, 0)); + + return protocol; +} + +int test_server(const char *hostname) +{ + TLSContext context; + std::auto_ptr conn = open_conn(hostname, context); + std::auto_ptr apProtocol( + test_server_login(*conn)); + BackupProtocolClient& protocol(*apProtocol); + // Make some test attributes #define ATTR1_SIZE 245 #define ATTR2_SIZE 23 @@ -911,28 +948,11 @@ // BLOCK { - // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); - - // Make a protocol - BackupProtocolClient protocol(conn); - // Get it logging FILE *protocolLog = ::fopen("testfiles/protocol.log", "w"); TEST_THAT(protocolLog != 0); protocol.SetLogToFile(protocolLog); - // Check the version - std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - - // Login - std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)); - - // Check marker is 0 - TEST_THAT(loginConf->GetClientStoreMarker() == 0); - #ifndef WIN32 // Check that we can't open a new connection which requests write permissions { @@ -1435,10 +1455,18 @@ } //} skip: + + std::auto_ptr apAccounts( + BackupStoreAccountDatabase::Read( + "testfiles/accounts.txt")); + std::auto_ptr apRefCount( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); // Create some nice recursive directories int64_t dirtodelete = create_test_data_subdirs(protocol, - BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */); + BackupProtocolClientListDirectory::RootDirectory, + "test_delete", 6 /* depth */, *apRefCount); // And delete them { @@ -1760,7 +1788,33 @@ TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); // make sure something is written to it + + std::auto_ptr apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr apReferences( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); + TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, + apReferences->GetLastObjectIDUsed()); + TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) + apReferences.reset(); + + // Delete the refcount database and log in again, + // check that it is recreated automatically but with + // no objects in it, to ensure seamless upgrade. + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); + + // Context + TLSContext context; + std::auto_ptr conn = open_conn("localhost", + context); + test_server_login(*conn)->QueryFinished(); + + apReferences = BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true); + TEST_EQUAL(0, apReferences->GetLastObjectIDUsed()); + TEST_THAT(ServerIsAlive(pid)); TEST_THAT(test_server("localhost") == 0); From boxbackup-dev at boxbackup.org Sun Jun 28 20:05:08 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:05:08 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2536 - box/trunk/lib/common Message-ID: <20090628190508.5AD9A325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:05:07 +0100 (Sun, 28 Jun 2009) New Revision: 2536 Modified: box/trunk/lib/common/NamedLock.cpp box/trunk/lib/common/NamedLock.h Log: Make NamedLock take a std::string instead of a char pointer for C++ style. Modified: box/trunk/lib/common/NamedLock.cpp =================================================================== --- box/trunk/lib/common/NamedLock.cpp 2009-06-28 12:04:25 UTC (rev 2535) +++ box/trunk/lib/common/NamedLock.cpp 2009-06-28 19:05:07 UTC (rev 2536) @@ -65,7 +65,7 @@ // Created: 2003/08/28 // // -------------------------------------------------------------------------- -bool NamedLock::TryAndGetLock(const char *Filename, int mode) +bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) { // Check if(mFileDescriptor != -1) @@ -75,7 +75,8 @@ // See if the lock can be got #if HAVE_DECL_O_EXLOCK - int fd = ::open(Filename, O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); + int fd = ::open(rFilename.c_str(), + O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); if(fd != -1) { // Got a lock, lovely @@ -92,10 +93,10 @@ return false; #else - int fd = ::open(Filename, O_WRONLY | O_CREAT | O_TRUNC, mode); + int fd = ::open(rFilename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode); if(fd == -1) { - BOX_WARNING("Failed to open lockfile: " << Filename); + BOX_WARNING("Failed to open lockfile: " << rFilename); THROW_EXCEPTION(CommonException, OSFileError) } Modified: box/trunk/lib/common/NamedLock.h =================================================================== --- box/trunk/lib/common/NamedLock.h 2009-06-28 12:04:25 UTC (rev 2535) +++ box/trunk/lib/common/NamedLock.h 2009-06-28 19:05:07 UTC (rev 2536) @@ -28,7 +28,7 @@ NamedLock(const NamedLock &); public: - bool TryAndGetLock(const char *Filename, int mode = 0755); + bool TryAndGetLock(const std::string& rFilename, int mode = 0755); bool GotLock() {return mFileDescriptor != -1;} void ReleaseLock(); From boxbackup-dev at boxbackup.org Sun Jun 28 20:22:24 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:22:24 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2537 - box/trunk/lib/common Message-ID: <20090628192224.B47FF325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:22:24 +0100 (Sun, 28 Jun 2009) New Revision: 2537 Modified: box/trunk/lib/common/BoxPortsAndFiles.h.in Log: Add a #define for the Box Backup test port, 22011. Modified: box/trunk/lib/common/BoxPortsAndFiles.h.in =================================================================== --- box/trunk/lib/common/BoxPortsAndFiles.h.in 2009-06-28 19:05:07 UTC (rev 2536) +++ box/trunk/lib/common/BoxPortsAndFiles.h.in 2009-06-28 19:22:24 UTC (rev 2537) @@ -15,6 +15,7 @@ // Backup store daemon #define BOX_PORT_BBSTORED (BOX_PORT_BASE+1) +#define BOX_PORT_BBSTORED_TEST 22011 // directory within the RAIDFILE root for the backup store daemon #define BOX_RAIDFILE_ROOT_BBSTORED "backup" From boxbackup-dev at boxbackup.org Sun Jun 28 20:24:02 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:24:02 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2538 - box/trunk/test/backupstore Message-ID: <20090628192402.D9D0A325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:24:02 +0100 (Sun, 28 Jun 2009) New Revision: 2538 Added: box/trunk/test/backupstore/Makefile.extra Log: Link testbackupstore against HousekeepStoreAccount so that it can call housekeeping directly. Added: box/trunk/test/backupstore/Makefile.extra =================================================================== --- box/trunk/test/backupstore/Makefile.extra (rev 0) +++ box/trunk/test/backupstore/Makefile.extra 2009-06-28 19:24:02 UTC (rev 2538) @@ -0,0 +1 @@ +link-extra: ../../bin/bbstored/HousekeepStoreAccount.o From boxbackup-dev at boxbackup.org Sun Jun 28 20:25:25 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:25:25 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2539 - box/trunk/lib/backupstore Message-ID: <20090628192525.71CE9325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:25:25 +0100 (Sun, 28 Jun 2009) New Revision: 2539 Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h Log: Make GetRefCount return a refcount_t rather than an int32_t. Make refcount_t unsigned and make its definition public. Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp 2009-06-28 19:24:02 UTC (rev 2538) +++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp 2009-06-28 19:25:25 UTC (rev 2539) @@ -247,7 +247,8 @@ // Created: 2009/06/01 // // -------------------------------------------------------------------------- -int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const +BackupStoreRefCountDatabase::refcount_t +BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const { IOStream::pos_type offset = GetOffset(ObjectID); Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h 2009-06-28 19:24:02 UTC (rev 2538) +++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h 2009-06-28 19:25:25 UTC (rev 2539) @@ -52,6 +52,8 @@ { friend class BackupStoreCheck; friend class BackupStoreContext; + friend class HousekeepStoreAccount; + public: ~BackupStoreRefCountDatabase(); private: @@ -73,8 +75,10 @@ static std::auto_ptr Load(const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + typedef uint32_t refcount_t; + // Data access functions - int32_t GetRefCount(int64_t ObjectID) const; + refcount_t GetRefCount(int64_t ObjectID) const; int64_t GetLastObjectIDUsed() const; // Data modification functions @@ -94,8 +98,6 @@ static void Create(const BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite); - typedef int32_t refcount_t; - static std::string GetFilename(const BackupStoreAccountDatabase::Entry& rAccount); IOStream::pos_type GetSize() const From boxbackup-dev at boxbackup.org Sun Jun 28 20:26:19 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:26:19 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2540 - box/trunk/lib/common Message-ID: <20090628192619.65D6E325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:26:19 +0100 (Sun, 28 Jun 2009) New Revision: 2540 Modified: box/trunk/lib/common/Test.h Log: Allow std::ostringstream formatting on the "line" part of TEST_EQUAL_LINE, so that it can be used to report object IDs for example. Modified: box/trunk/lib/common/Test.h =================================================================== --- box/trunk/lib/common/Test.h 2009-06-28 19:25:25 UTC (rev 2539) +++ box/trunk/lib/common/Test.h 2009-06-28 19:26:19 UTC (rev 2540) @@ -113,7 +113,9 @@ \ if(_exp_str != _found_str) \ { \ - std::string _line_str = _line; \ + std::ostringstream _ossl; \ + _ossl << _line; \ + std::string _line_str = _ossl.str(); \ printf("Expected <%s> but found <%s> in <%s>\n", \ _exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \ \ From boxbackup-dev at boxbackup.org Sun Jun 28 20:29:11 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:29:11 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2541 - box/trunk/bin/bbstored Message-ID: <20090628192911.08D74325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:29:10 +0100 (Sun, 28 Jun 2009) New Revision: 2541 Modified: box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp box/trunk/bin/bbstored/BackupStoreDaemon.h box/trunk/bin/bbstored/HousekeepStoreAccount.cpp box/trunk/bin/bbstored/HousekeepStoreAccount.h Log: Make housekeeping check the object reference counts and fix them if they're wrong. Add a callback interface to decouple housekeeping from the BackupStoreDaemon, allowing it to be called directly in tests. Allow housekeeping callers to request it to keep trying forever to get a lock on the account if it's busy. Modified: box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp =================================================================== --- box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp 2009-06-28 19:26:19 UTC (rev 2540) +++ box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp 2009-06-28 19:29:10 UTC (rev 2541) @@ -108,7 +108,8 @@ mpAccounts->GetAccountRoot(*i, rootDir, discSet); // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); + HousekeepStoreAccount housekeeping(*i, rootDir, + discSet, this); housekeeping.DoHousekeeping(); } } Modified: box/trunk/bin/bbstored/BackupStoreDaemon.h =================================================================== --- box/trunk/bin/bbstored/BackupStoreDaemon.h 2009-06-28 19:26:19 UTC (rev 2540) +++ box/trunk/bin/bbstored/BackupStoreDaemon.h 2009-06-28 19:29:10 UTC (rev 2541) @@ -14,11 +14,11 @@ #include "BoxPortsAndFiles.h" #include "BackupConstants.h" #include "BackupStoreContext.h" +#include "HousekeepStoreAccount.h" #include "IOStreamGetLine.h" class BackupStoreAccounts; class BackupStoreAccountDatabase; -class HousekeepStoreAccount; // -------------------------------------------------------------------------- // @@ -29,10 +29,8 @@ // // -------------------------------------------------------------------------- class BackupStoreDaemon : public ServerTLS, - HousekeepingInterface + HousekeepingInterface, HousekeepingCallback { - friend class HousekeepStoreAccount; - public: BackupStoreDaemon(); ~BackupStoreDaemon(); @@ -64,10 +62,13 @@ // Housekeeping functions void HousekeepingProcess(); - bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); void LogConnectionStats(const char *commonName, const SocketStreamTLS &s); +public: + // HousekeepingInterface implementation + virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); + private: BackupStoreAccountDatabase *mpAccountDatabase; BackupStoreAccounts *mpAccounts; Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.cpp =================================================================== --- box/trunk/bin/bbstored/HousekeepStoreAccount.cpp 2009-06-28 19:26:19 UTC (rev 2540) +++ box/trunk/bin/bbstored/HousekeepStoreAccount.cpp 2009-06-28 19:29:10 UTC (rev 2541) @@ -39,11 +39,13 @@ // Created: 11/12/03 // // -------------------------------------------------------------------------- -HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon) +HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, + const std::string &rStoreRoot, int StoreDiscSet, + HousekeepingCallback* pHousekeepingCallback) : mAccountID(AccountID), mStoreRoot(rStoreRoot), mStoreDiscSet(StoreDiscSet), - mrDaemon(rDaemon), + mpHousekeepingCallback(pHousekeepingCallback), mDeletionSizeTarget(0), mPotentialDeletionsTotalSize(0), mMaxSizeInPotentialDeletions(0), @@ -57,6 +59,7 @@ mBlocksInDirectoriesDelta(0), mFilesDeleted(0), mEmptyDirectoriesDeleted(0), + mSuppressRefCountChangeWarnings(false), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { } @@ -81,7 +84,7 @@ // Created: 11/12/03 // // -------------------------------------------------------------------------- -void HousekeepStoreAccount::DoHousekeeping() +void HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) { // Attempt to lock the account std::string writeLockFilename; @@ -91,8 +94,21 @@ if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */)) { - // Couldn't lock the account -- just stop now - return; + if(KeepTryingForever) + { + BOX_WARNING("Failed to lock account for housekeeping, " + "still trying..."); + while(!writeLock.TryAndGetLock(writeLockFilename, + 0600 /* restrictive file permissions */)) + { + sleep(1); + } + } + else + { + // Couldn't lock the account -- just stop now + return; + } } // Load the store info to find necessary info for the housekeeping @@ -106,6 +122,14 @@ mDeletionSizeTarget = 0; } + // initialise the refcount database + mNewRefCounts.clear(); + // try to pre-allocate as much memory as we need + mNewRefCounts.reserve(info->GetLastObjectIDUsed()); + // initialise the refcount of the root entry + mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0); + mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1; + // Scan the directory for potential things to delete // This will also remove eligible items marked with RemoveASAP bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); @@ -207,6 +231,86 @@ mEmptyDirectoriesDeleted << " dirs)" << (deleteInterrupted?" and was interrupted":"")); } + + // We can only update the refcount database if we successfully + // finished our scan of all directories, otherwise we don't actually + // know which of the new counts are valid and which aren't + // (we might not have seen second references to some objects, etc.) + + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + std::auto_ptr apReferences; + + // try to load the reference count database + try + { + apReferences = BackupStoreRefCountDatabase::Load(account, + false); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database is missing or corrupted " + "during housekeeping, creating a new one."); + mSuppressRefCountChangeWarnings = true; + BackupStoreRefCountDatabase::CreateForRegeneration(account); + apReferences = BackupStoreRefCountDatabase::Load(account, + false); + } + + int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed(); + + for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; + ObjectID < mNewRefCounts.size(); ObjectID++) + { + if (ObjectID > LastUsedObjectIdOnDisk) + { + if (!mSuppressRefCountChangeWarnings) + { + BOX_WARNING("Reference count of object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " not found in database, added" + " with " << mNewRefCounts[ObjectID] << + " references"); + } + apReferences->SetRefCount(ObjectID, + mNewRefCounts[ObjectID]); + LastUsedObjectIdOnDisk = ObjectID; + continue; + } + + BackupStoreRefCountDatabase::refcount_t OldRefCount = + apReferences->GetRefCount(ObjectID); + + if (OldRefCount != mNewRefCounts[ObjectID]) + { + BOX_WARNING("Reference count of object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " changed from " << OldRefCount << + " to " << mNewRefCounts[ObjectID]); + apReferences->SetRefCount(ObjectID, + mNewRefCounts[ObjectID]); + } + } + + // zero excess references in the database + for (int64_t ObjectID = mNewRefCounts.size(); + ObjectID <= LastUsedObjectIdOnDisk; ObjectID++) + { + BackupStoreRefCountDatabase::refcount_t OldRefCount = + apReferences->GetRefCount(ObjectID); + BackupStoreRefCountDatabase::refcount_t NewRefCount = 0; + + if (OldRefCount != NewRefCount) + { + BOX_WARNING("Reference count of object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " changed from " << OldRefCount << + " to " << NewRefCount << " (not found)"); + apReferences->SetRefCount(ObjectID, NewRefCount); + } + } + + // force file to be saved and closed before releasing the lock below + apReferences.reset(); // Make sure the delta's won't cause problems if the counts are // really wrong, and it wasn't fixed because the store was @@ -279,7 +383,7 @@ // Check for having to stop // Include account ID here as the specified account is locked - if(mrDaemon.CheckForInterProcessMsg(mAccountID)) + if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) { // Need to abort now return false; @@ -359,6 +463,13 @@ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) { + // This directory references this object + if (mNewRefCounts.size() <= en->GetObjectID()) + { + mNewRefCounts.resize(en->GetObjectID() + 1, 0); + } + mNewRefCounts[en->GetObjectID()]++; + // Update recalculated usage sizes int16_t enFlags = en->GetFlags(); int64_t enSizeInBlocks = en->GetSizeInBlocks(); @@ -467,6 +578,13 @@ BackupStoreDirectory::Entry *en = 0; while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) { + // This parent directory references this child + if (mNewRefCounts.size() <= en->GetObjectID()) + { + mNewRefCounts.resize(en->GetObjectID() + 1, 0); + } + mNewRefCounts[en->GetObjectID()]++; + // Next level ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir); @@ -551,7 +669,7 @@ { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; // Check for having to stop - if(mrDaemon.CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked + if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked { // Need to abort now return true; @@ -808,7 +926,7 @@ { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; // Check for having to stop - if(mrDaemon.CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked + if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked { // Need to abort now return true; Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.h =================================================================== --- box/trunk/bin/bbstored/HousekeepStoreAccount.h 2009-06-28 19:26:19 UTC (rev 2540) +++ box/trunk/bin/bbstored/HousekeepStoreAccount.h 2009-06-28 19:29:10 UTC (rev 2541) @@ -17,6 +17,12 @@ class BackupStoreDaemon; class BackupStoreDirectory; +class HousekeepingCallback +{ + public: + virtual ~HousekeepingCallback() {} + virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0; +}; // -------------------------------------------------------------------------- // @@ -29,10 +35,11 @@ class HousekeepStoreAccount { public: - HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon); + HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, + int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback); ~HousekeepStoreAccount(); - void DoHousekeeping(); + void DoHousekeeping(bool KeepTryingForever = false); private: @@ -65,7 +72,7 @@ int mAccountID; std::string mStoreRoot; int mStoreDiscSet; - BackupStoreDaemon &mrDaemon; + HousekeepingCallback* mpHousekeepingCallback; int64_t mDeletionSizeTarget; @@ -91,6 +98,10 @@ // Deletion count int64_t mFilesDeleted; int64_t mEmptyDirectoriesDeleted; + + // New reference count list + std::vector mNewRefCounts; + bool mSuppressRefCountChangeWarnings; // Poll frequency int mCountUntilNextInterprocessMsgCheck; From boxbackup-dev at boxbackup.org Sun Jun 28 20:33:35 2009 From: boxbackup-dev at boxbackup.org (boxbackup-dev at boxbackup.org) Date: Sun, 28 Jun 2009 20:33:35 +0100 (BST) Subject: [Box Backup-commit] COMMIT r2542 - in box/trunk/test/backupstore: . testfiles Message-ID: <20090628193335.0AB78325020@www.boxbackup.org> Author: chris Date: 2009-06-28 20:33:34 +0100 (Sun, 28 Jun 2009) New Revision: 2542 Modified: box/trunk/test/backupstore/testbackupstore.cpp box/trunk/test/backupstore/testfiles/bbstored.conf Log: Test that housekeeping can recreate the refcount database if it's missing. Modified: box/trunk/test/backupstore/testbackupstore.cpp =================================================================== --- box/trunk/test/backupstore/testbackupstore.cpp 2009-06-28 19:29:10 UTC (rev 2541) +++ box/trunk/test/backupstore/testbackupstore.cpp 2009-06-28 19:33:34 UTC (rev 2542) @@ -36,6 +36,8 @@ #include "ServerControl.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreRefCountDatabase.h" +#include "BackupStoreAccounts.h" +#include "HousekeepStoreAccount.h" #include "MemLeakFindOn.h" @@ -472,6 +474,17 @@ TEST_THAT(dirs == 0 || dirs == 2); } +std::vector ExpectedRefCounts; + +void set_refcount(int64_t ObjectID, uint32_t RefCount = 1) +{ + if (ExpectedRefCounts.size() <= ObjectID); + { + ExpectedRefCounts.resize(ObjectID + 1, 0); + } + ExpectedRefCounts[ObjectID] = RefCount; +} + void create_file_in_dir(std::string name, std::string source, int64_t parentId, BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount) { @@ -489,6 +502,7 @@ int64_t objectId = stored->GetObjectID(); TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed()); TEST_EQUAL(1, rRefCount.GetRefCount(objectId)) + set_refcount(objectId, 1); } int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, @@ -509,13 +523,9 @@ printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid); - std::auto_ptr apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - std::auto_ptr apReferences( - BackupStoreRefCountDatabase::Load( - apAccounts->GetEntry(0x1234567), true)); TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) + set_refcount(subdirid, 1); // Put more directories in it, if we haven't gone down too far if(depth > 0) @@ -634,7 +644,8 @@ // Get a connection SocketStreamTLS connReadOnly; - connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); + connReadOnly.Open(context, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); BackupProtocolClient protocolReadOnly(connReadOnly); { @@ -796,6 +807,7 @@ store1objid = stored->GetObjectID(); TEST_THAT(store1objid == 2); } + set_refcount(store1objid, 1); // And retrieve it { // Retrieve as object @@ -902,7 +914,8 @@ { init_context(rContext); std::auto_ptr conn(new SocketStreamTLS); - conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); + conn->Open(rContext, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); return conn; } @@ -924,6 +937,17 @@ return protocol; } +void run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + int discSet = rAccount.GetDiscSet(); + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, + discSet, NULL); + housekeeping.DoHousekeeping(true /* keep trying forever */); +} + int test_server(const char *hostname) { TLSContext context; @@ -957,7 +981,8 @@ // Check that we can't open a new connection which requests write permissions { SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); BackupProtocolClient protocol(conn); std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); @@ -973,7 +998,8 @@ #ifndef WIN32 // Open a new connection which is read only SocketStreamTLS connReadOnly; - connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); + connReadOnly.Open(context, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); BackupProtocolClient protocolReadOnly(connReadOnly); // Get it logging @@ -1021,6 +1047,10 @@ uploads[t].allocated_objid = stored->GetObjectID(); uploads[t].mod_time = modtime; if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); + set_refcount(stored->GetObjectID(), 1); + BOX_TRACE("wrote file " << filename << " to server " + "as object " << + BOX_FORMAT_OBJECTID(stored->GetObjectID())); } // Add some attributes onto one of them @@ -1141,6 +1171,9 @@ if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); patchedID = stored->GetObjectID(); } + + set_refcount(patchedID, 1); + // Then download it to check it's OK std::auto_ptr getFile(protocol.QueryGetFile(BackupProtocolClientListDirectory::RootDirectory, patchedID)); TEST_THAT(getFile->GetObjectID() == patchedID); @@ -1162,6 +1195,9 @@ subdirid = dirCreate->GetObjectID(); TEST_THAT(subdirid == maxID + 1); } + + set_refcount(subdirid, 1); + // Stick a file in it int64_t subdirfileid = 0; { @@ -1179,6 +1215,8 @@ subdirfileid = stored->GetObjectID(); } + set_refcount(subdirfileid, 1); + printf("\n==== Checking upload using read-only connection\n"); // Check the directories on the read only connection { @@ -1402,6 +1440,9 @@ subsubfileid = stored->GetObjectID(); } + set_refcount(subsubdirid, 1); + set_refcount(subsubfileid, 1); + // Query names -- test that invalid stuff returns not found OK { std::auto_ptr nameRep(protocol.QueryGetObjectName(3248972347823478927LL, subsubdirid)); @@ -1759,7 +1800,8 @@ { // Open a connection to the server SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, "localhost", + BOX_PORT_BBSTORED_TEST); // Make a protocol BackupProtocolClient protocol(conn); @@ -1805,19 +1847,61 @@ // no objects in it, to ensure seamless upgrade. TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); - // Context TLSContext context; std::auto_ptr conn = open_conn("localhost", context); test_server_login(*conn)->QueryFinished(); - apReferences = BackupStoreRefCountDatabase::Load( - apAccounts->GetEntry(0x1234567), true); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + apReferences = BackupStoreRefCountDatabase::Load(account, true); TEST_EQUAL(0, apReferences->GetLastObjectIDUsed()); TEST_THAT(ServerIsAlive(pid)); + run_housekeeping(account); + + // Check that housekeeping fixed the ref counts + TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, + apReferences->GetLastObjectIDUsed()); + TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) + + TEST_THAT(ServerIsAlive(pid)); + + set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + TEST_THAT(test_server("localhost") == 0); + + // test that all object reference counts have the + // expected values + TEST_EQUAL(ExpectedRefCounts.size() - 1, + apReferences->GetLastObjectIDUsed()); + for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; + i < ExpectedRefCounts.size(); i++) + { + TEST_EQUAL_LINE(ExpectedRefCounts[i], + apReferences->GetRefCount(i), + "object " << BOX_FORMAT_OBJECTID(i)); + } + + // Delete the refcount database again, and let + // housekeeping recreate it and fix the ref counts. + // This could also happen after upgrade, if a housekeeping + // runs before the user logs in. + apReferences.reset(); + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); + run_housekeeping(account); + apReferences = BackupStoreRefCountDatabase::Load(account, true); + + TEST_EQUAL(ExpectedRefCounts.size() - 1, + apReferences->GetLastObjectIDUsed()); + for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; + i < ExpectedRefCounts.size(); i++) + { + TEST_EQUAL_LINE(ExpectedRefCounts[i], + apReferences->GetRefCount(i), + "object " << BOX_FORMAT_OBJECTID(i)); + } // Test the deletion of objects by the housekeeping system // First, things as they are now. @@ -1886,7 +1970,8 @@ { // Open a connection to the server SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, "localhost", + BOX_PORT_BBSTORED_TEST); // Make a protocol BackupProtocolClient protocol(conn); Modified: box/trunk/test/backupstore/testfiles/bbstored.conf =================================================================== --- box/trunk/test/backupstore/testfiles/bbstored.conf 2009-06-28 19:29:10 UTC (rev 2541) +++ box/trunk/test/backupstore/testfiles/bbstored.conf 2009-06-28 19:33:34 UTC (rev 2542) @@ -9,7 +9,7 @@ Server { PidFile = testfiles/bbstored.pid - ListenAddresses = inet:localhost + ListenAddresses = inet:localhost:22011 CertificateFile = testfiles/serverCerts.pem PrivateKeyFile = testfiles/serverPrivKey.pem TrustedCAsFile = testfiles/serverTrustedCAs.pem