From subversion at boxbackup.org  Fri Jun 29 23:15:39 2012
From: subversion at boxbackup.org (subversion at boxbackup.org)
Date: Fri, 29 Jun 2012 23:15:39 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r3114 - in box/trunk:
	bin/bbstoreaccounts bin/bbstored lib/backupstore
Message-ID: <201206292215.q5TMFdPg068312@wm.boxbackup.org>

Author: chris
Date: 2012-06-29 23:15:36 +0100 (Fri, 29 Jun 2012)
New Revision: 3114

Added:
   box/trunk/lib/backupstore/HousekeepStoreAccount.cpp
   box/trunk/lib/backupstore/HousekeepStoreAccount.h
Removed:
   box/trunk/bin/bbstored/HousekeepStoreAccount.cpp
   box/trunk/bin/bbstored/HousekeepStoreAccount.h
Modified:
   box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp
Log:
Add housekeep command to bbstoreaccounts to run housekeeping right now.


Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp
===================================================================
--- box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp	2012-06-11 21:13:08 UTC (rev 3113)
+++ box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp	2012-06-29 22:15:36 UTC (rev 3114)
@@ -24,17 +24,18 @@
 #include <ostream>
 #include <vector>
 
-#include "BoxPortsAndFiles.h"
-#include "BackupStoreConfigVerify.h"
-#include "RaidFileController.h"
 #include "BackupStoreAccounts.h"
 #include "BackupStoreAccountDatabase.h"
+#include "BackupStoreCheck.h"
+#include "BackupStoreConfigVerify.h"
+#include "BackupStoreInfo.h"
+#include "BoxPortsAndFiles.h"
+#include "HousekeepStoreAccount.h"
 #include "MainHelper.h"
-#include "BackupStoreInfo.h"
+#include "NamedLock.h"
+#include "RaidFileController.h"
 #include "StoreStructure.h"
-#include "NamedLock.h"
 #include "UnixUser.h"
-#include "BackupStoreCheck.h"
 #include "Utils.h"
 
 #include "MemLeakFindOn.h"
@@ -264,7 +265,6 @@
 	return 0;
 }
 
-
 int AccountInfo(Configuration &rConfig, int32_t ID)
 {
 	// Load in the account database 
@@ -438,7 +438,9 @@
 	return retcode;
 }
 
-int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool FixErrors, bool Quiet)
+bool OpenAccount(Configuration &rConfig, int32_t ID, 
+	const std::string &rUsername, std::string &rRootDirOut,
+	int &rDiscSetOut, std::auto_ptr<UnixUser> apUser)
 {
 	// Load in the account database 
 	std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str()));
@@ -448,25 +450,39 @@
 	{
 		BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << 
 			" does not exist.");
-		return 1;
+		return false;
 	}
 	
 	// Get info from the database
 	BackupStoreAccounts acc(*db);
-	std::string rootDir;
-	int discSetNum;
-	acc.GetAccountRoot(ID, rootDir, discSetNum);
+	acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut);
 	
 	// Become the right user
-	std::auto_ptr<UnixUser> user;
 	if(!rUsername.empty())
 	{
 		// Username specified, change...
-		user.reset(new UnixUser(rUsername.c_str()));
-		user->ChangeProcessUser(true /* temporary */);
-		// Change will be undone at the end of this function
+		apUser.reset(new UnixUser(rUsername));
+		apUser->ChangeProcessUser(true /* temporary */);
+		// Change will be undone when apUser goes out of scope
+		// in the caller.
 	}
 
+	return true;
+}
+
+int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool FixErrors, bool Quiet)
+{
+	std::string rootDir;
+	int discSetNum;
+	std::auto_ptr<UnixUser> user;
+
+	if(!OpenAccount(rConfig, ID, rUsername, rootDir, discSetNum, user))
+	{
+		BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
+			<< " for checking.");
+		return 1;
+	}
+
 	// Check it
 	BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet);
 	check.Check();
@@ -496,6 +512,38 @@
 	return 0;
 }
 
+int HousekeepAccountNow(Configuration &rConfig, const std::string &rUsername,
+	int32_t ID)
+{
+	std::string rootDir;
+	int discSetNum;
+	std::auto_ptr<UnixUser> user;
+
+	if(!OpenAccount(rConfig, ID, rUsername, rootDir, discSetNum, user))
+	{
+		BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
+			<< " for housekeeping.");
+		return 1;
+	}
+
+	HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL);
+	bool success = housekeeping.DoHousekeeping();
+
+	if(!success)
+	{
+		BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID)
+			<< " for housekeeping: perhaps a client is "
+			"still connected?");
+		return 1;
+	}
+	else
+	{
+		BOX_TRACE("Finished housekeeping on account " <<
+			BOX_FORMAT_ACCOUNT(ID));
+		return 0;
+	}
+}
+
 void PrintUsageAndExit()
 {
 	printf(
@@ -526,6 +574,10 @@
 "        Changes the \"name\" of the account to the specified string.\n"
 "        The name is purely cosmetic and intended to make it easier to\n"
 "        identify your accounts.\n"
+"  housekeep <account>\n"
+"        Runs housekeeping immediately on the account. If it cannot be locked,\n"
+"        bbstoreaccounts returns an error status code (1), otherwise success\n"
+"        (0) even if any errors were fixed by housekeeping.\n"
 	);
 	exit(2);
 }
@@ -706,6 +758,10 @@
 		// Check the account
 		return CheckAccount(*config, username, id, fixErrors, quiet);
 	}
+	else if(::strcmp(argv[0], "housekeep") == 0)
+	{
+		return HousekeepAccountNow(*config, username, id);
+	}
 	else
 	{
 		BOX_ERROR("Unknown command '" << argv[0] << "'.");

Deleted: box/trunk/bin/bbstored/HousekeepStoreAccount.cpp
===================================================================
--- box/trunk/bin/bbstored/HousekeepStoreAccount.cpp	2012-06-11 21:13:08 UTC (rev 3113)
+++ box/trunk/bin/bbstored/HousekeepStoreAccount.cpp	2012-06-29 22:15:36 UTC (rev 3114)
@@ -1,1101 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-//		Name:    HousekeepStoreAccount.cpp
-//		Purpose: 
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include <stdio.h>
-
-#include <map>
-
-#include "HousekeepStoreAccount.h"
-#include "BackupStoreDaemon.h"
-#include "StoreStructure.h"
-#include "BackupStoreConstants.h"
-#include "RaidFileRead.h"
-#include "RaidFileWrite.h"
-#include "BackupStoreDirectory.h"
-#include "BackupStoreInfo.h"
-#include "NamedLock.h"
-#include "autogen_BackupStoreException.h"
-#include "BackupStoreFile.h"
-#include "BufferedStream.h"
-
-#include "MemLeakFindOn.h"
-
-// check every 32 directories scanned/files deleted
-#define POLL_INTERPROCESS_MSG_CHECK_FREQUENCY	32
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::HousekeepStoreAccount(int, const std::string &, int, BackupStoreDaemon &)
-//		Purpose: Constructor
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
-	const std::string &rStoreRoot, int StoreDiscSet,
-	HousekeepingCallback* pHousekeepingCallback)
-	: mAccountID(AccountID),
-	  mStoreRoot(rStoreRoot),
-	  mStoreDiscSet(StoreDiscSet),
-	  mpHousekeepingCallback(pHousekeepingCallback),
-	  mDeletionSizeTarget(0),
-  	  mPotentialDeletionsTotalSize(0),
-	  mMaxSizeInPotentialDeletions(0),
-	  mBlocksUsed(0),
-	  mBlocksInOldFiles(0),
-	  mBlocksInDeletedFiles(0),
-	  mBlocksInDirectories(0),
-	  mBlocksUsedDelta(0),
-	  mBlocksInOldFilesDelta(0),
-	  mBlocksInDeletedFilesDelta(0),
-	  mBlocksInDirectoriesDelta(0),
-	  mFilesDeleted(0),
-	  mEmptyDirectoriesDeleted(0),
-	  mSuppressRefCountChangeWarnings(false),
-	  mRefCountsAdjusted(0),
-	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::~HousekeepStoreAccount()
-//		Purpose: Destructor
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-HousekeepStoreAccount::~HousekeepStoreAccount()
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::DoHousekeeping()
-//		Purpose: Perform the housekeeping
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-void HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
-{
-	BOX_TRACE("Starting housekeeping on account " <<
-		BOX_FORMAT_OBJECTID(mAccountID));
-
-	// Attempt to lock the account
-	std::string writeLockFilename;
-	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet,
-		writeLockFilename);
-	NamedLock writeLock;
-	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
-		0600 /* restrictive file permissions */))
-	{
-		if(KeepTryingForever)
-		{
-			BOX_WARNING("Failed to lock account for housekeeping, "
-				"still trying...");
-			while(!writeLock.TryAndGetLock(writeLockFilename,
-				0600 /* restrictive file permissions */))
-			{
-				sleep(1);
-			}
-		}
-		else
-		{
-			// Couldn't lock the account -- just stop now
-			return;
-		}
-	}
-
-	// Load the store info to find necessary info for the housekeeping
-	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID,
-		mStoreRoot, mStoreDiscSet, false /* Read/Write */));
-	std::auto_ptr<BackupStoreInfo> pOldInfo(
-		BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet,
-			true /* Read Only */));
-
-	// Tag log output to identify account
-	std::ostringstream tag;
-	tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" <<
-		info->GetAccountName();
-	Logging::Tagger tagWithClientID(tag.str());
-
-	// Calculate how much should be deleted
-	mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
-	if(mDeletionSizeTarget < 0)
-	{
-		mDeletionSizeTarget = 0;
-	}
-
-	// initialise the refcount database
-	mNewRefCounts.clear();
-	// try to pre-allocate as much memory as we need
-	mNewRefCounts.reserve(info->GetLastObjectIDUsed());
-	// initialise the refcount of the root entry
-	mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
-	mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
-
-	// Scan the directory for potential things to delete
-	// This will also remove eligible items marked with RemoveASAP
-	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
-
-	// If scan directory stopped for some reason, probably parent
-	// instructed to terminate, stop now.
-	if(!continueHousekeeping)
-	{
-		// If any files were marked "delete now", then update
-		// the size of the store.
-		if(mBlocksUsedDelta != 0 || 
-			mBlocksInOldFilesDelta != 0 ||
-			mBlocksInDeletedFilesDelta != 0)
-		{
-			info->ChangeBlocksUsed(mBlocksUsedDelta);
-			info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
-			info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
-
-			// Save the store info back
-			info->ReportChangesTo(*pOldInfo);
-			info->Save();
-		}
-	
-		return;
-	}
-
-	// Log any difference in opinion between the values recorded in
-	// the store info, and the values just calculated for space usage.
-	// BLOCK
-	{
-		int64_t used = info->GetBlocksUsed();
-		int64_t usedOld = info->GetBlocksInOldFiles();
-		int64_t usedDeleted = info->GetBlocksInDeletedFiles();
-		int64_t usedDirectories = info->GetBlocksInDirectories();
-
-		// If the counts were wrong, taking into account RemoveASAP 
-		// items deleted, log a message
-		if((used + mBlocksUsedDelta) != mBlocksUsed 
-			|| (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
-			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles 
-			|| usedDirectories != mBlocksInDirectories)
-		{
-			// Log this
-			BOX_ERROR("Housekeeping on account " << 
-				BOX_FORMAT_ACCOUNT(mAccountID) << " found "
-				"and fixed wrong block counts: "
-				"used (" <<
-				(used + mBlocksUsedDelta) << "," <<
-				mBlocksUsed << "), old (" <<
-				(usedOld + mBlocksInOldFilesDelta) << "," <<
-				mBlocksInOldFiles << "), deleted (" <<
-				(usedDeleted + mBlocksInDeletedFilesDelta) <<
-				"," << mBlocksInDeletedFiles << "), dirs (" <<
-				usedDirectories << "," << mBlocksInDirectories
-				<< ")");
-		}
-		
-		// If the current values don't match, store them
-		if(used != mBlocksUsed 
-			|| usedOld != mBlocksInOldFiles
-			|| usedDeleted != mBlocksInDeletedFiles 
-			|| usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
-		{	
-			// Set corrected values in store info
-			info->CorrectAllUsedValues(mBlocksUsed,
-				mBlocksInOldFiles, mBlocksInDeletedFiles,
-				mBlocksInDirectories + mBlocksInDirectoriesDelta);
-
-			info->ReportChangesTo(*pOldInfo);
-			info->Save();
-		}
-	}
-	
-	// Reset the delta counts for files, as they will include 
-	// RemoveASAP flagged files deleted during the initial scan.
-
-	// keep for reporting
-	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
-
-	mBlocksUsedDelta = 0;
-	mBlocksInOldFilesDelta = 0;
-	mBlocksInDeletedFilesDelta = 0;
-	
-	// Go and delete items from the accounts
-	bool deleteInterrupted = DeleteFiles();
-	
-	// If that wasn't interrupted, remove any empty directories which 
-	// are also marked as deleted in their containing directory
-	if(!deleteInterrupted)
-	{
-		deleteInterrupted = DeleteEmptyDirectories();
-	}
-	
-	// Log deletion if anything was deleted
-	if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0)
-	{
-		BOX_INFO("Housekeeping on account " << 
-			BOX_FORMAT_ACCOUNT(mAccountID) << " "
-			"removed " <<
-			(0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) <<
-			" blocks (" << mFilesDeleted << " files, " <<
-			mEmptyDirectoriesDeleted << " dirs)" <<
-			(deleteInterrupted?" and was interrupted":""));
-	}
-
-	// We can only update the refcount database if we successfully
-	// finished our scan of all directories, otherwise we don't actually
-	// know which of the new counts are valid and which aren't
-	// (we might not have seen second references to some objects, etc.)
-
-	BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
-	std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
-
-	// try to load the reference count database
-	try
-	{
-		apReferences = BackupStoreRefCountDatabase::Load(account,
-			false);
-	}
-	catch(BoxException &e)
-	{
-		BOX_WARNING("Reference count database is missing or corrupted "
-			"during housekeeping, creating a new one.");
-		mSuppressRefCountChangeWarnings = true;
-		BackupStoreRefCountDatabase::CreateForRegeneration(account);
-		apReferences = BackupStoreRefCountDatabase::Load(account,
-			false);
-	}
-
-	int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
-
-	for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
-		ObjectID < mNewRefCounts.size(); ObjectID++)
-	{
-		if (ObjectID > LastUsedObjectIdOnDisk)
-		{
-			if (!mSuppressRefCountChangeWarnings)
-			{
-				BOX_WARNING("Reference count of object " <<
-					BOX_FORMAT_OBJECTID(ObjectID) <<
-					" not found in database, added"
-					" with " << mNewRefCounts[ObjectID] <<
-					" references");
-			}
-			apReferences->SetRefCount(ObjectID,
-				mNewRefCounts[ObjectID]);
-			mRefCountsAdjusted++;
-			LastUsedObjectIdOnDisk = ObjectID;
-			continue;
-		}
-
-		BackupStoreRefCountDatabase::refcount_t OldRefCount =
-			apReferences->GetRefCount(ObjectID);
-
-		if (OldRefCount != mNewRefCounts[ObjectID])
-		{
-			BOX_WARNING("Reference count of object " <<
-				BOX_FORMAT_OBJECTID(ObjectID) <<
-				" changed from " << OldRefCount <<
-				" to " << mNewRefCounts[ObjectID]);
-			apReferences->SetRefCount(ObjectID,
-				mNewRefCounts[ObjectID]);
-			mRefCountsAdjusted++;
-		}
-	}
-
-	// zero excess references in the database
-	for (int64_t ObjectID = mNewRefCounts.size(); 
-		ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
-	{
-		BackupStoreRefCountDatabase::refcount_t OldRefCount =
-			apReferences->GetRefCount(ObjectID);
-		BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
-
-		if (OldRefCount != NewRefCount)
-		{
-			BOX_WARNING("Reference count of object " <<
-				BOX_FORMAT_OBJECTID(ObjectID) <<
-				" changed from " << OldRefCount <<
-				" to " << NewRefCount << " (not found)");
-			apReferences->SetRefCount(ObjectID, NewRefCount);
-			mRefCountsAdjusted++;
-		}
-	}
-
-	// force file to be saved and closed before releasing the lock below
-	apReferences.reset();
-	
-	// Make sure the delta's won't cause problems if the counts are 
-	// really wrong, and it wasn't fixed because the store was 
-	// updated during the scan.
-	if(mBlocksUsedDelta < (0 - info->GetBlocksUsed()))
-	{
-		mBlocksUsedDelta = (0 - info->GetBlocksUsed());
-	}
-	if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles()))
-	{
- 		mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles());
-	}
-	if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles()))
-	{
-	 	mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles());
-	}
-	if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories()))
-	{
-		mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories());
-	}
-	
-	// Update the usage counts in the store
-	info->ChangeBlocksUsed(mBlocksUsedDelta);
-	info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
-	info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
-	info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
-	
-	// Save the store info back
-	info->ReportChangesTo(*pOldInfo);
-	info->Save();
-	
-	// Explicity release the lock (would happen automatically on 
-	// going out of scope, included for code clarity)
-	writeLock.ReleaseLock();
-
-	BOX_TRACE("Finished housekeeping on account " <<
-		BOX_FORMAT_OBJECTID(mAccountID));
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::MakeObjectFilename(int64_t, std::string &)
-//		Purpose: Generate and place the filename for a given object ID
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut)
-{
-	// Delegate to utility function
-	StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rFilenameOut, false /* don't bother ensuring the directory exists */);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
-//		Purpose: Private. Scan a directory for potentially deleteable
-//			 items, and add them to the list. Returns true if the
-//			 scan should continue.
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
-{
-#ifndef WIN32
-	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
-	{
-		mCountUntilNextInterprocessMsgCheck =
-			POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
-
-		// Check for having to stop
-		// Include account ID here as the specified account is locked
-		if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
-		{
-			// Need to abort now
-			return false;
-		}
-	}
-#endif
-
-	// Get the filename
-	std::string objectFilename;
-	MakeObjectFilename(ObjectID, objectFilename);
-
-	// Open it.
-	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet,
-		objectFilename));
-	
-	// Add the size of the directory on disc to the size being calculated
-	int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
-	mBlocksInDirectories += originalDirSizeInBlocks;
-	mBlocksUsed += originalDirSizeInBlocks;
-	
-	// Read the directory in
-	BackupStoreDirectory dir;
-	BufferedStream buf(*dirStream);
-	dir.ReadFromStream(buf, IOStream::TimeOutInfinite);
-	dirStream->Close();
-	
-	// Is it empty?
-	if(dir.GetNumberOfEntries() == 0)
-	{
-		// Add it to the list of directories to potentially delete
-		mEmptyDirectories.push_back(dir.GetObjectID());
-	}
-
-	// Calculate reference counts first, before we start requesting
-	// files to be deleted.
-	// BLOCK
-	{
-		BackupStoreDirectory::Iterator i(dir);
-		BackupStoreDirectory::Entry *en = 0;
-
-		while((en = i.Next()) != 0)
-		{
-			// This directory references this object
-			if (mNewRefCounts.size() <= en->GetObjectID())
-			{
-				mNewRefCounts.resize(en->GetObjectID() + 1, 0);
-			}
-			mNewRefCounts[en->GetObjectID()]++;
-		}
-	}
-
-	// BLOCK
-	{
-		// Remove any files which are marked for removal as soon
-		// as they become old or deleted.
-		bool deletedSomething = false;
-		do
-		{
-			// Iterate through the directory
-			deletedSomething = false;
-			BackupStoreDirectory::Iterator i(dir);
-			BackupStoreDirectory::Entry *en = 0;
-			while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
-			{
-				int16_t enFlags = en->GetFlags();
-				if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
-					&& (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
-				{
-					// Delete this immediately.
-					DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
-					
-					// flag as having done something
-					deletedSomething = true;
-					
-					// Must start the loop from the beginning again, as iterator is now
-					// probably invalid.
-					break;
-				}
-			}
-		} while(deletedSomething);
-	}
-	
-	// BLOCK
-	{
-		// Add files to the list of potential deletions
-
-		// map to count the distance from the mark
-		typedef std::pair<std::string, int32_t> version_t;
-		std::map<version_t, int32_t> markVersionAges;
-			// map of pair (filename, mark number) -> version age
-
-		// NOTE: use a reverse iterator to allow the distance from mark stuff to work
-		BackupStoreDirectory::ReverseIterator i(dir);
-		BackupStoreDirectory::Entry *en = 0;
-
-		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
-		{
-			// Update recalculated usage sizes
-			int16_t enFlags = en->GetFlags();
-			int64_t enSizeInBlocks = en->GetSizeInBlocks();
-			mBlocksUsed += enSizeInBlocks;
-			if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks;
-			if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks;
-					
-			// Work out ages of this version from the last mark
-			int32_t enVersionAge = 0;
-			std::map<version_t, int32_t>::iterator enVersionAgeI(
-				markVersionAges.find(
-					version_t(en->GetName().GetEncodedFilename(),
-						en->GetMarkNumber())));
-			if(enVersionAgeI != markVersionAges.end())
-			{
-				enVersionAge = enVersionAgeI->second + 1;
-				enVersionAgeI->second = enVersionAge;
-			}
-			else
-			{
-				markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge;
-			}
-			// enVersionAge is now the age of this version.
-			
-			// Potentially add it to the list if it's deleted, if it's an old version or deleted
-			if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
-			{
-				// Is deleted / old version.
-				DelEn d;
-				d.mObjectID = en->GetObjectID();
-				d.mInDirectory = ObjectID;
-				d.mSizeInBlocks = en->GetSizeInBlocks();
-				d.mMarkNumber = en->GetMarkNumber();
-				d.mVersionAgeWithinMark = enVersionAge;
-				d.mIsFlagDeleted = (enFlags &
-					BackupStoreDirectory::Entry::Flags_Deleted)
-					? true : false;
-				
-				// Add it to the list
-				mPotentialDeletions.insert(d);
-				
-				// Update various counts
-				mPotentialDeletionsTotalSize += d.mSizeInBlocks;
-				if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks;
-				
-				// Too much in the list of potential deletions?
-				// (check against the deletion target + the max size in deletions, so that we never delete things
-				// and take the total size below the deletion size target)
-				if(mPotentialDeletionsTotalSize > (mDeletionSizeTarget + mMaxSizeInPotentialDeletions))
-				{
-					int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions);
-					bool recalcMaxSize = false;
-					
-					while(sizeToRemove > 0)
-					{
-						// Make iterator for the last element, while checking that there's something there in the first place.
-						std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.end());
-						if(i != mPotentialDeletions.begin())
-						{
-							// Nothing left in set
-							break;
-						}
-						// Make this into an iterator pointing to the last element in the set
-						--i;
-						
-						// Delete this one?
-						if(sizeToRemove > i->mSizeInBlocks)
-						{
-							sizeToRemove -= i->mSizeInBlocks;
-							if(i->mSizeInBlocks >= mMaxSizeInPotentialDeletions)
-							{
-								// Will need to recalculate the maximum size now, because we've just deleted that element
-								recalcMaxSize = true;
-							}
-							mPotentialDeletions.erase(i);
-						}
-						else
-						{
-							// Over the size to remove, so stop now
-							break;
-						}
-					}
-					
-					if(recalcMaxSize)
-					{
-						// Because an object which was the maximum size recorded was deleted from the set
-						// it's necessary to recalculate this maximum.
-						mMaxSizeInPotentialDeletions = 0;
-						std::set<DelEn, DelEnCompare>::const_iterator i(mPotentialDeletions.begin());
-						for(; i != mPotentialDeletions.end(); ++i)
-						{
-							if(i->mSizeInBlocks > mMaxSizeInPotentialDeletions)
-							{
-								mMaxSizeInPotentialDeletions = i->mSizeInBlocks;
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-
-	{
-		// Recurse into subdirectories
-		BackupStoreDirectory::Iterator i(dir);
-		BackupStoreDirectory::Entry *en = 0;
-		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
-		{
-			// Next level
-			ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
-			
-			if(!ScanDirectory(en->GetObjectID()))
-			{
-				// Halting operation
-				return false;
-			}
-		}
-	}
-	
-	return true;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &, const HousekeepStoreAccount::DelEnd &)
-//		Purpose: Comparison function for set
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y)
-{
-	// STL spec says this:
-	// A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. 
-
-	// The sort order here is intended to preserve the entries of most value, that is, the newest objects
-	// which are on a mark boundary.
-	
-	// Reverse order age, so oldest goes first
-	if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark)
-	{
-		return true;
-	}
-	else if(x.mVersionAgeWithinMark < y.mVersionAgeWithinMark)
-	{
-		return false;
-	}
-
-	// but mark number in ascending order, so that the oldest marks are deleted first
-	if(x.mMarkNumber < y.mMarkNumber)
-	{
-		return true;
-	}
-	else if(x.mMarkNumber > y.mMarkNumber)
-	{
-		return false;
-	}
-
-	// Just compare object ID now to put the oldest objects first
-	return x.mObjectID < y.mObjectID;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::DeleteFiles()
-//		Purpose: Delete the files targeted for deletion, returning
-//			 true if the operation was interrupted
-//		Created: 15/12/03
-//
-// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::DeleteFiles()
-{
-	// Only delete files if the deletion target is greater than zero
-	// (otherwise we delete one file each time round, which gradually deletes the old versions)
-	if(mDeletionSizeTarget <= 0)
-	{
-		// Not interrupted
-		return false;
-	}
-
-	// Iterate through the set of potential deletions, until enough has been deleted.
-	// (there is likely to be more in the set than should be actually deleted).
-	for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i)
-	{
-#ifndef WIN32
-		if((--mCountUntilNextInterprocessMsgCheck) <= 0)
-		{
-			mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
-			// Check for having to stop
-			if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
-			{
-				// Need to abort now
-				return true;
-			}
-		}
-#endif
-
-		// Load up the directory it's in
-		// Get the filename
-		std::string dirFilename;
-		BackupStoreDirectory dir;
-		int64_t dirSizeInBlocksOrig = 0;
-		{
-			MakeObjectFilename(i->mInDirectory, dirFilename);
-			std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
-			dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks();
-			dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
-		}
-		
-		// Delete the file
-		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
-		BOX_INFO("Housekeeping removed " <<
-			(i->mIsFlagDeleted ? "deleted" : "old") <<
-			" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
-			" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
-		
-		// Stop if the deletion target has been matched or exceeded
-		// (checking here rather than at the beginning will tend to reduce the
-		// space to slightly less than the soft limit, which will allow the backup
-		// client to start uploading files again)
-		if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget)
-		{
-			break;
-		}
-	}
-
-	return false;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::DeleteFile(int64_t, int64_t,
-//			 BackupStoreDirectory &, const std::string &, int64_t)
-//		Purpose: Delete a file. Takes the directory already loaded
-//			 in and the filename, for efficiency in both the
-//			 usage scenarios.
-//		Created: 15/7/04
-//
-// --------------------------------------------------------------------------
-void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
-{
-	// Find the entry inside the directory
-	bool wasDeleted = false;
-	bool wasOldVersion = false;
-	int64_t deletedFileSizeInBlocks = 0;
-	// A pointer to an object which requires committing if the directory save goes OK
-	std::auto_ptr<RaidFileWrite> padjustedEntry;
-	// BLOCK
-	{
-		BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
-		if(pentry == 0)
-		{
-			BOX_ERROR("Housekeeping on account " <<
-				BOX_FORMAT_ACCOUNT(mAccountID) << " "
-				"found error: object " <<
-				BOX_FORMAT_OBJECTID(ObjectID) << " "
-				"not found in dir " << 
-				BOX_FORMAT_OBJECTID(InDirectory) << ", "
-				"indicates logic error/corruption? Run "
-				"bbstoreaccounts check <accid> fix");
-			return;
-		}
-		
-		// Record the flags it's got set
-		wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
-		wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
-		// Check this should be deleted
-		if(!wasDeleted && !wasOldVersion)
-		{
-			// Things changed size we were last around
-			return;
-		}
-		
-		// Record size
-		deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
-		
-		// If the entry is involved in a chain of patches, it needs to be handled
-		// a bit more carefully.
-		if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
-		{
-			// This entry is a patch from a newer entry. Just need to update the info on that entry.
-			BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
-			if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
-			{
-				THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
-			}
-			// Change the info in the newer entry so that this no longer points to this entry
-			pnewer->SetDependsOlder(0);
-		}
-		else if(pentry->GetDependsOlder() != 0)
-		{
-			BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
-			if(pentry->GetDependsNewer() == 0)
-			{
-				// There exists an older version which depends on this one. Need to combine the two over that one.
-
-				// Adjust the other entry in the directory
-				if(polder == 0 || polder->GetDependsNewer() != ObjectID)
-				{
-					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
-				}
-				// Change the info in the older entry so that this no longer points to this entry
-				polder->SetDependsNewer(0);
-			}
-			else
-			{
-				// This entry is in the middle of a chain, and two patches need combining.
-				
-				// First, adjust the directory entries
-				BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
-				if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
-					|| polder == 0 || polder->GetDependsNewer() != ObjectID)
-				{
-					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
-				}
-				// Remove the middle entry from the linked list by simply using the values from this entry
-				pnewer->SetDependsOlder(pentry->GetDependsOlder());
-				polder->SetDependsNewer(pentry->GetDependsNewer());
-			}
-			
-			// COMMON CODE to both cases
-
-			// Generate the filename of the older version
-			std::string objFilenameOlder;
-			MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
-			// Open it twice (it's the diff)
-			std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
-			std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
-			// Open this file
-			std::string objFilename;
-			MakeObjectFilename(ObjectID, objFilename);
-			std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
-			// And open a write file to overwrite the other directory entry
-			padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
-				objFilenameOlder, mNewRefCounts[ObjectID]));
-			padjustedEntry->Open(true /* allow overwriting */);
-
-			if(pentry->GetDependsNewer() == 0)
-			{
-				// There exists an older version which depends on this one. Need to combine the two over that one.
-				BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
-			}
-			else
-			{
-				// This entry is in the middle of a chain, and two patches need combining.
-				BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
-			}
-			// The file will be committed later when the directory is safely commited.
-			
-			// Work out the adjusted size
-			int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
-			int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
-			mBlocksUsedDelta += sizeDelta;
-			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
-			{
-				mBlocksInDeletedFilesDelta += sizeDelta;
-			}
-			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
-			{
-				mBlocksInOldFilesDelta += sizeDelta;
-			}
-			polder->SetSizeInBlocks(newSize);
-		}
-		
-		// pentry no longer valid
-	}
-	
-	// Delete it from the directory
-	rDirectory.DeleteEntry(ObjectID);
-	
-	// Save directory back to disc
-	// BLOCK
-	int64_t dirRevisedSize = 0;
-	{
-		RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
-			mNewRefCounts[InDirectory]);
-		writeDir.Open(true /* allow overwriting */);
-		rDirectory.WriteToStream(writeDir);
-
-		// get the disc usage (must do this before commiting it)
-		dirRevisedSize = writeDir.GetDiscUsageInBlocks();
-
-		// Commit directory
-		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
-
-		// adjust usage counts for this directory
-		if(dirRevisedSize > 0)
-		{
-			int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
-			mBlocksUsedDelta += adjust;
-			mBlocksInDirectoriesDelta += adjust;
-		}
-	}
-
-	// Commit any new adjusted entry
-	if(padjustedEntry.get() != 0)
-	{
-		padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
-		padjustedEntry.reset();	// delete it now
-	}
-
-	// Drop reference count by one. If it reaches zero, delete the file.
-	if(--mNewRefCounts[ObjectID] == 0)
-	{
-		// Delete from disc
-		BOX_TRACE("Removing unreferenced object " <<
-			BOX_FORMAT_OBJECTID(ObjectID));
-		std::string objFilename;
-		MakeObjectFilename(ObjectID, objFilename);
-		RaidFileWrite del(mStoreDiscSet, objFilename,
-			mNewRefCounts[ObjectID]);
-		del.Delete();
-	}
-	else
-	{
-		BOX_TRACE("Preserving object " <<
-			BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
-			mNewRefCounts[ObjectID] << " references");
-	}
-
-	// Adjust counts for the file
-	++mFilesDeleted;
-	mBlocksUsedDelta -= deletedFileSizeInBlocks;
-	if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
-	if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
-	
-	// Delete the directory?
-	// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
-	if(rDirectory.GetNumberOfEntries() == 0)
-	{
-		// Candidate for deletion
-		mEmptyDirectories.push_back(InDirectory);
-	}
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//		Name:    HousekeepStoreAccount::DeleteEmptyDirectories()
-//		Purpose: Remove any empty directories which are also marked as deleted in their containing directory,
-//				 returning true if the opertaion was interrupted
-//		Created: 15/12/03
-//
-// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::DeleteEmptyDirectories()
-{
-	while(mEmptyDirectories.size() > 0)
-	{
-		std::vector<int64_t> toExamine;
-
-		// Go through list
-		for(std::vector<int64_t>::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i)
-		{
-#ifndef WIN32
-			if((--mCountUntilNextInterprocessMsgCheck) <= 0)
-			{
-				mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
-				// Check for having to stop
-				if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
-				{
-					// Need to abort now
-					return true;
-				}
-			}
-#endif
-
-			// Do not delete the root directory
-			if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID)
-			{
-				continue;
-			}
-
-			DeleteEmptyDirectory(*i, toExamine);
-		}
-
-		// Remove contents of empty directories
-		mEmptyDirectories.clear();
-		// Swap in new, so it's examined next time round
-		mEmptyDirectories.swap(toExamine);
-	}
-	
-	// Not interrupted
-	return false;
-}
-
-void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
-	std::vector<int64_t>& rToExamine)
-{
-	// Load up the directory to potentially delete
-	std::string dirFilename;
-	BackupStoreDirectory dir;
-	int64_t dirSizeInBlocks = 0;
-
-	// BLOCK
-	{
-		MakeObjectFilename(dirId, dirFilename);
-		// Check it actually exists (just in case it gets
-		// added twice to the list)
-		if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
-		{
-			// doesn't exist, next!
-			return;
-		}
-		// load
-		std::auto_ptr<RaidFileRead> dirStream(
-			RaidFileRead::Open(mStoreDiscSet, dirFilename));
-		dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
-		dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
-	}
-
-	// Make sure this directory is actually empty
-	if(dir.GetNumberOfEntries() != 0)
-	{
-		// Not actually empty, try next one
-		return;
-	}
-
-	// Candidate for deletion... open containing directory
-	std::string containingDirFilename;
-	BackupStoreDirectory containingDir;
-	int64_t containingDirSizeInBlocksOrig = 0;
-	{
-		MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
-		std::auto_ptr<RaidFileRead> containingDirStream(
-			RaidFileRead::Open(mStoreDiscSet,
-				containingDirFilename));
-		containingDirSizeInBlocksOrig = 
-			containingDirStream->GetDiscUsageInBlocks();
-		containingDir.ReadFromStream(*containingDirStream,
-			IOStream::TimeOutInfinite);
-	}
-
-	// Find the entry
-	BackupStoreDirectory::Entry *pdirentry = 
-		containingDir.FindEntryByID(dir.GetObjectID());
-	if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
-	{
-		// Should be deleted
-		containingDir.DeleteEntry(dir.GetObjectID());
-
-		// Is the containing dir now a candidate for deletion?
-		if(containingDir.GetNumberOfEntries() == 0)
-		{
-			rToExamine.push_back(containingDir.GetObjectID());
-		}
-
-		// Write revised parent directory
-		RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
-			mNewRefCounts[containingDir.GetObjectID()]);
-		writeDir.Open(true /* allow overwriting */);
-		containingDir.WriteToStream(writeDir);
-
-		// get the disc usage (must do this before commiting it)
-		int64_t dirSize = writeDir.GetDiscUsageInBlocks();
-
-		// Commit directory
-		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
-
-		// adjust usage counts for this directory
-		if(dirSize > 0)
-		{
-			int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
-			mBlocksUsedDelta += adjust;
-			mBlocksInDirectoriesDelta += adjust;
-		}
-
-		
-		if (--mNewRefCounts[dir.GetObjectID()] == 0)
-		{
-			// Delete the directory itself
-			RaidFileWrite del(mStoreDiscSet, dirFilename,
-				mNewRefCounts[dir.GetObjectID()]);
-			del.Delete();
-		}
-
-		BOX_INFO("Housekeeping removed empty deleted dir " <<
-			BOX_FORMAT_OBJECTID(dirId));
-
-		// And adjust usage counts for the directory that's
-		// just been deleted
-		mBlocksUsedDelta -= dirSizeInBlocks;
-		mBlocksInDirectoriesDelta -= dirSizeInBlocks;
-
-		// Update count
-		++mEmptyDirectoriesDeleted;
-	}
-}
-

Deleted: box/trunk/bin/bbstored/HousekeepStoreAccount.h
===================================================================
--- box/trunk/bin/bbstored/HousekeepStoreAccount.h	2012-06-11 21:13:08 UTC (rev 3113)
+++ box/trunk/bin/bbstored/HousekeepStoreAccount.h	2012-06-29 22:15:36 UTC (rev 3114)
@@ -1,112 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-//		Name:    HousekeepStoreAccount.h
-//		Purpose: Action class to perform housekeeping on a store account
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-
-#ifndef HOUSEKEEPSTOREACCOUNT__H
-#define HOUSEKEEPSTOREACCOUNT__H
-
-#include <string>
-#include <set>
-#include <vector>
-
-class BackupStoreDaemon;
-class BackupStoreDirectory;
-
-class HousekeepingCallback
-{
-	public:
-	virtual ~HousekeepingCallback() {}
-	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0;
-};
-
-// --------------------------------------------------------------------------
-//
-// Class
-//		Name:    HousekeepStoreAccount
-//		Purpose: Action class to perform housekeeping on a store account
-//		Created: 11/12/03
-//
-// --------------------------------------------------------------------------
-class HousekeepStoreAccount
-{
-public:
-	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot,
-		int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback);
-	~HousekeepStoreAccount();
-	
-	void DoHousekeeping(bool KeepTryingForever = false);
-	int GetRefCountsAdjusted() { return mRefCountsAdjusted; }
-	
-private:
-	// utility functions
-	void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
-
-	bool ScanDirectory(int64_t ObjectID);
-	bool DeleteFiles();
-	bool DeleteEmptyDirectories();
-	void DeleteEmptyDirectory(int64_t dirId,
-		std::vector<int64_t>& rToExamine);
-	void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
-
-private:
-	typedef struct
-	{
-		int64_t mObjectID;
-		int64_t mInDirectory;
-		int64_t mSizeInBlocks;
-		int32_t mMarkNumber;
-		int32_t mVersionAgeWithinMark;	// 0 == current, 1 latest old version, etc
-		bool    mIsFlagDeleted; // false for files flagged "Old"
-	} DelEn;
-	
-	struct DelEnCompare
-	{
-		bool operator()(const DelEn &x, const DelEn &y);
-	};
-	
-	int mAccountID;
-	std::string mStoreRoot;
-	int mStoreDiscSet;
-	HousekeepingCallback* mpHousekeepingCallback;
-	
-	int64_t mDeletionSizeTarget;
-	
-	std::set<DelEn, DelEnCompare> mPotentialDeletions;
-	int64_t mPotentialDeletionsTotalSize;
-	int64_t mMaxSizeInPotentialDeletions;
-	
-	// List of directories which are empty, and might be good for deleting
-	std::vector<int64_t> mEmptyDirectories;
-	
-	// The re-calculated blocks used stats
-	int64_t mBlocksUsed;
-	int64_t mBlocksInOldFiles;
-	int64_t mBlocksInDeletedFiles;
-	int64_t mBlocksInDirectories;
-
-	// Deltas from deletion
-	int64_t mBlocksUsedDelta;
-	int64_t mBlocksInOldFilesDelta;
-	int64_t mBlocksInDeletedFilesDelta;
-	int64_t mBlocksInDirectoriesDelta;
-	
-	// Deletion count
-	int64_t mFilesDeleted;
-	int64_t mEmptyDirectoriesDeleted;
-
-	// New reference count list
-	std::vector<uint32_t> mNewRefCounts;
-	bool mSuppressRefCountChangeWarnings;
-	int mRefCountsAdjusted;
-	
-	// Poll frequency
-	int mCountUntilNextInterprocessMsgCheck;
-};
-
-#endif // HOUSEKEEPSTOREACCOUNT__H
-

Copied: box/trunk/lib/backupstore/HousekeepStoreAccount.cpp (from rev 3113, box/trunk/bin/bbstored/HousekeepStoreAccount.cpp)
===================================================================
--- box/trunk/lib/backupstore/HousekeepStoreAccount.cpp	                        (rev 0)
+++ box/trunk/lib/backupstore/HousekeepStoreAccount.cpp	2012-06-29 22:15:36 UTC (rev 3114)
@@ -0,0 +1,1104 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    HousekeepStoreAccount.cpp
+//		Purpose: 
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include <map>
+
+#include "autogen_BackupStoreException.h"
+#include "BackupConstants.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreRefCountDatabase.h"
+#include "BufferedStream.h"
+#include "HousekeepStoreAccount.h"
+#include "NamedLock.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "StoreStructure.h"
+
+#include "MemLeakFindOn.h"
+
+// check every 32 directories scanned/files deleted
+#define POLL_INTERPROCESS_MSG_CHECK_FREQUENCY	32
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::HousekeepStoreAccount(int, const std::string &, int, BackupStoreDaemon &)
+//		Purpose: Constructor
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
+	const std::string &rStoreRoot, int StoreDiscSet,
+	HousekeepingCallback* pHousekeepingCallback)
+	: mAccountID(AccountID),
+	  mStoreRoot(rStoreRoot),
+	  mStoreDiscSet(StoreDiscSet),
+	  mpHousekeepingCallback(pHousekeepingCallback),
+	  mDeletionSizeTarget(0),
+  	  mPotentialDeletionsTotalSize(0),
+	  mMaxSizeInPotentialDeletions(0),
+	  mBlocksUsed(0),
+	  mBlocksInOldFiles(0),
+	  mBlocksInDeletedFiles(0),
+	  mBlocksInDirectories(0),
+	  mBlocksUsedDelta(0),
+	  mBlocksInOldFilesDelta(0),
+	  mBlocksInDeletedFilesDelta(0),
+	  mBlocksInDirectoriesDelta(0),
+	  mFilesDeleted(0),
+	  mEmptyDirectoriesDeleted(0),
+	  mSuppressRefCountChangeWarnings(false),
+	  mRefCountsAdjusted(0),
+	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::~HousekeepStoreAccount()
+//		Purpose: Destructor
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::~HousekeepStoreAccount()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DoHousekeeping()
+//		Purpose: Perform the housekeeping
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
+{
+	BOX_TRACE("Starting housekeeping on account " <<
+		BOX_FORMAT_ACCOUNT(mAccountID));
+
+	// Attempt to lock the account
+	std::string writeLockFilename;
+	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet,
+		writeLockFilename);
+	NamedLock writeLock;
+	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
+		0600 /* restrictive file permissions */))
+	{
+		if(KeepTryingForever)
+		{
+			BOX_WARNING("Failed to lock account for housekeeping, "
+				"still trying...");
+			while(!writeLock.TryAndGetLock(writeLockFilename,
+				0600 /* restrictive file permissions */))
+			{
+				sleep(1);
+			}
+		}
+		else
+		{
+			// Couldn't lock the account -- just stop now
+			return false;
+		}
+	}
+
+	// Load the store info to find necessary info for the housekeeping
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID,
+		mStoreRoot, mStoreDiscSet, false /* Read/Write */));
+	std::auto_ptr<BackupStoreInfo> pOldInfo(
+		BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet,
+			true /* Read Only */));
+
+	// Tag log output to identify account
+	std::ostringstream tag;
+	tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" <<
+		info->GetAccountName();
+	Logging::Tagger tagWithClientID(tag.str());
+
+	// Calculate how much should be deleted
+	mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
+	if(mDeletionSizeTarget < 0)
+	{
+		mDeletionSizeTarget = 0;
+	}
+
+	// initialise the refcount database
+	mNewRefCounts.clear();
+	// try to pre-allocate as much memory as we need
+	mNewRefCounts.reserve(info->GetLastObjectIDUsed());
+	// initialise the refcount of the root entry
+	mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
+	mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+
+	// Scan the directory for potential things to delete
+	// This will also remove eligible items marked with RemoveASAP
+	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+	// If scan directory stopped for some reason, probably parent
+	// instructed to terminate, stop now.
+	if(!continueHousekeeping)
+	{
+		// If any files were marked "delete now", then update
+		// the size of the store.
+		if(mBlocksUsedDelta != 0 || 
+			mBlocksInOldFilesDelta != 0 ||
+			mBlocksInDeletedFilesDelta != 0)
+		{
+			info->ChangeBlocksUsed(mBlocksUsedDelta);
+			info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+			info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+
+			// Save the store info back
+			info->ReportChangesTo(*pOldInfo);
+			info->Save();
+		}
+	
+		return false;
+	}
+
+	// Log any difference in opinion between the values recorded in
+	// the store info, and the values just calculated for space usage.
+	// BLOCK
+	{
+		int64_t used = info->GetBlocksUsed();
+		int64_t usedOld = info->GetBlocksInOldFiles();
+		int64_t usedDeleted = info->GetBlocksInDeletedFiles();
+		int64_t usedDirectories = info->GetBlocksInDirectories();
+
+		// If the counts were wrong, taking into account RemoveASAP 
+		// items deleted, log a message
+		if((used + mBlocksUsedDelta) != mBlocksUsed 
+			|| (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
+			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles 
+			|| usedDirectories != mBlocksInDirectories)
+		{
+			// Log this
+			BOX_ERROR("Housekeeping on account " << 
+				BOX_FORMAT_ACCOUNT(mAccountID) << " found "
+				"and fixed wrong block counts: "
+				"used (" <<
+				(used + mBlocksUsedDelta) << "," <<
+				mBlocksUsed << "), old (" <<
+				(usedOld + mBlocksInOldFilesDelta) << "," <<
+				mBlocksInOldFiles << "), deleted (" <<
+				(usedDeleted + mBlocksInDeletedFilesDelta) <<
+				"," << mBlocksInDeletedFiles << "), dirs (" <<
+				usedDirectories << "," << mBlocksInDirectories
+				<< ")");
+		}
+		
+		// If the current values don't match, store them
+		if(used != mBlocksUsed 
+			|| usedOld != mBlocksInOldFiles
+			|| usedDeleted != mBlocksInDeletedFiles 
+			|| usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
+		{	
+			// Set corrected values in store info
+			info->CorrectAllUsedValues(mBlocksUsed,
+				mBlocksInOldFiles, mBlocksInDeletedFiles,
+				mBlocksInDirectories + mBlocksInDirectoriesDelta);
+
+			info->ReportChangesTo(*pOldInfo);
+			info->Save();
+		}
+	}
+	
+	// Reset the delta counts for files, as they will include 
+	// RemoveASAP flagged files deleted during the initial scan.
+
+	// keep for reporting
+	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+
+	mBlocksUsedDelta = 0;
+	mBlocksInOldFilesDelta = 0;
+	mBlocksInDeletedFilesDelta = 0;
+	
+	// Go and delete items from the accounts
+	bool deleteInterrupted = DeleteFiles();
+	
+	// If that wasn't interrupted, remove any empty directories which 
+	// are also marked as deleted in their containing directory
+	if(!deleteInterrupted)
+	{
+		deleteInterrupted = DeleteEmptyDirectories();
+	}
+	
+	// Log deletion if anything was deleted
+	if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0)
+	{
+		BOX_INFO("Housekeeping on account " << 
+			BOX_FORMAT_ACCOUNT(mAccountID) << " "
+			"removed " <<
+			(0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) <<
+			" blocks (" << mFilesDeleted << " files, " <<
+			mEmptyDirectoriesDeleted << " dirs)" <<
+			(deleteInterrupted?" and was interrupted":""));
+	}
+
+	// We can only update the refcount database if we successfully
+	// finished our scan of all directories, otherwise we don't actually
+	// know which of the new counts are valid and which aren't
+	// (we might not have seen second references to some objects, etc.)
+
+	BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+	std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
+
+	// try to load the reference count database
+	try
+	{
+		apReferences = BackupStoreRefCountDatabase::Load(account,
+			false);
+	}
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database is missing or corrupted "
+			"during housekeeping, creating a new one.");
+		mSuppressRefCountChangeWarnings = true;
+		BackupStoreRefCountDatabase::CreateForRegeneration(account);
+		apReferences = BackupStoreRefCountDatabase::Load(account,
+			false);
+	}
+
+	int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
+
+	for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+		ObjectID < mNewRefCounts.size(); ObjectID++)
+	{
+		if (ObjectID > LastUsedObjectIdOnDisk)
+		{
+			if (!mSuppressRefCountChangeWarnings)
+			{
+				BOX_WARNING("Reference count of object " <<
+					BOX_FORMAT_OBJECTID(ObjectID) <<
+					" not found in database, added"
+					" with " << mNewRefCounts[ObjectID] <<
+					" references");
+			}
+			apReferences->SetRefCount(ObjectID,
+				mNewRefCounts[ObjectID]);
+			mRefCountsAdjusted++;
+			LastUsedObjectIdOnDisk = ObjectID;
+			continue;
+		}
+
+		BackupStoreRefCountDatabase::refcount_t OldRefCount =
+			apReferences->GetRefCount(ObjectID);
+
+		if (OldRefCount != mNewRefCounts[ObjectID])
+		{
+			BOX_WARNING("Reference count of object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) <<
+				" changed from " << OldRefCount <<
+				" to " << mNewRefCounts[ObjectID]);
+			apReferences->SetRefCount(ObjectID,
+				mNewRefCounts[ObjectID]);
+			mRefCountsAdjusted++;
+		}
+	}
+
+	// zero excess references in the database
+	for (int64_t ObjectID = mNewRefCounts.size(); 
+		ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
+	{
+		BackupStoreRefCountDatabase::refcount_t OldRefCount =
+			apReferences->GetRefCount(ObjectID);
+		BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
+
+		if (OldRefCount != NewRefCount)
+		{
+			BOX_WARNING("Reference count of object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) <<
+				" changed from " << OldRefCount <<
+				" to " << NewRefCount << " (not found)");
+			apReferences->SetRefCount(ObjectID, NewRefCount);
+			mRefCountsAdjusted++;
+		}
+	}
+
+	// force file to be saved and closed before releasing the lock below
+	apReferences.reset();
+	
+	// Make sure the delta's won't cause problems if the counts are 
+	// really wrong, and it wasn't fixed because the store was 
+	// updated during the scan.
+	if(mBlocksUsedDelta < (0 - info->GetBlocksUsed()))
+	{
+		mBlocksUsedDelta = (0 - info->GetBlocksUsed());
+	}
+	if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles()))
+	{
+ 		mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles());
+	}
+	if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles()))
+	{
+	 	mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles());
+	}
+	if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories()))
+	{
+		mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories());
+	}
+	
+	// Update the usage counts in the store
+	info->ChangeBlocksUsed(mBlocksUsedDelta);
+	info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+	info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+	info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
+	
+	// Save the store info back
+	info->ReportChangesTo(*pOldInfo);
+	info->Save();
+	
+	// Explicity release the lock (would happen automatically on 
+	// going out of scope, included for code clarity)
+	writeLock.ReleaseLock();
+
+	BOX_TRACE("Finished housekeeping on account " <<
+		BOX_FORMAT_ACCOUNT(mAccountID));
+	return true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::MakeObjectFilename(int64_t, std::string &)
+//		Purpose: Generate and place the filename for a given object ID
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut)
+{
+	// Delegate to utility function
+	StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rFilenameOut, false /* don't bother ensuring the directory exists */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
+//		Purpose: Private. Scan a directory for potentially deleteable
+//			 items, and add them to the list. Returns true if the
+//			 scan should continue.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+{
+#ifndef WIN32
+	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+	{
+		mCountUntilNextInterprocessMsgCheck =
+			POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+
+		// Check for having to stop
+		// Include account ID here as the specified account is locked
+		if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
+		{
+			// Need to abort now
+			return false;
+		}
+	}
+#endif
+
+	// Get the filename
+	std::string objectFilename;
+	MakeObjectFilename(ObjectID, objectFilename);
+
+	// Open it.
+	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet,
+		objectFilename));
+	
+	// Add the size of the directory on disc to the size being calculated
+	int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+	mBlocksInDirectories += originalDirSizeInBlocks;
+	mBlocksUsed += originalDirSizeInBlocks;
+	
+	// Read the directory in
+	BackupStoreDirectory dir;
+	BufferedStream buf(*dirStream);
+	dir.ReadFromStream(buf, IOStream::TimeOutInfinite);
+	dirStream->Close();
+	
+	// Is it empty?
+	if(dir.GetNumberOfEntries() == 0)
+	{
+		// Add it to the list of directories to potentially delete
+		mEmptyDirectories.push_back(dir.GetObjectID());
+	}
+
+	// Calculate reference counts first, before we start requesting
+	// files to be deleted.
+	// BLOCK
+	{
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+
+		while((en = i.Next()) != 0)
+		{
+			// This directory references this object
+			if (mNewRefCounts.size() <= en->GetObjectID())
+			{
+				mNewRefCounts.resize(en->GetObjectID() + 1, 0);
+			}
+			mNewRefCounts[en->GetObjectID()]++;
+		}
+	}
+
+	// BLOCK
+	{
+		// Remove any files which are marked for removal as soon
+		// as they become old or deleted.
+		bool deletedSomething = false;
+		do
+		{
+			// Iterate through the directory
+			deletedSomething = false;
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+			{
+				int16_t enFlags = en->GetFlags();
+				if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
+					&& (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+				{
+					// Delete this immediately.
+					DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+					
+					// flag as having done something
+					deletedSomething = true;
+					
+					// Must start the loop from the beginning again, as iterator is now
+					// probably invalid.
+					break;
+				}
+			}
+		} while(deletedSomething);
+	}
+	
+	// BLOCK
+	{
+		// Add files to the list of potential deletions
+
+		// map to count the distance from the mark
+		typedef std::pair<std::string, int32_t> version_t;
+		std::map<version_t, int32_t> markVersionAges;
+			// map of pair (filename, mark number) -> version age
+
+		// NOTE: use a reverse iterator to allow the distance from mark stuff to work
+		BackupStoreDirectory::ReverseIterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+		{
+			// Update recalculated usage sizes
+			int16_t enFlags = en->GetFlags();
+			int64_t enSizeInBlocks = en->GetSizeInBlocks();
+			mBlocksUsed += enSizeInBlocks;
+			if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks;
+			if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks;
+					
+			// Work out ages of this version from the last mark
+			int32_t enVersionAge = 0;
+			std::map<version_t, int32_t>::iterator enVersionAgeI(
+				markVersionAges.find(
+					version_t(en->GetName().GetEncodedFilename(),
+						en->GetMarkNumber())));
+			if(enVersionAgeI != markVersionAges.end())
+			{
+				enVersionAge = enVersionAgeI->second + 1;
+				enVersionAgeI->second = enVersionAge;
+			}
+			else
+			{
+				markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge;
+			}
+			// enVersionAge is now the age of this version.
+			
+			// Potentially add it to the list if it's deleted, if it's an old version or deleted
+			if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+			{
+				// Is deleted / old version.
+				DelEn d;
+				d.mObjectID = en->GetObjectID();
+				d.mInDirectory = ObjectID;
+				d.mSizeInBlocks = en->GetSizeInBlocks();
+				d.mMarkNumber = en->GetMarkNumber();
+				d.mVersionAgeWithinMark = enVersionAge;
+				d.mIsFlagDeleted = (enFlags &
+					BackupStoreDirectory::Entry::Flags_Deleted)
+					? true : false;
+				
+				// Add it to the list
+				mPotentialDeletions.insert(d);
+				
+				// Update various counts
+				mPotentialDeletionsTotalSize += d.mSizeInBlocks;
+				if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks;
+				
+				// Too much in the list of potential deletions?
+				// (check against the deletion target + the max size in deletions, so that we never delete things
+				// and take the total size below the deletion size target)
+				if(mPotentialDeletionsTotalSize > (mDeletionSizeTarget + mMaxSizeInPotentialDeletions))
+				{
+					int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions);
+					bool recalcMaxSize = false;
+					
+					while(sizeToRemove > 0)
+					{
+						// Make iterator for the last element, while checking that there's something there in the first place.
+						std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.end());
+						if(i != mPotentialDeletions.begin())
+						{
+							// Nothing left in set
+							break;
+						}
+						// Make this into an iterator pointing to the last element in the set
+						--i;
+						
+						// Delete this one?
+						if(sizeToRemove > i->mSizeInBlocks)
+						{
+							sizeToRemove -= i->mSizeInBlocks;
+							if(i->mSizeInBlocks >= mMaxSizeInPotentialDeletions)
+							{
+								// Will need to recalculate the maximum size now, because we've just deleted that element
+								recalcMaxSize = true;
+							}
+							mPotentialDeletions.erase(i);
+						}
+						else
+						{
+							// Over the size to remove, so stop now
+							break;
+						}
+					}
+					
+					if(recalcMaxSize)
+					{
+						// Because an object which was the maximum size recorded was deleted from the set
+						// it's necessary to recalculate this maximum.
+						mMaxSizeInPotentialDeletions = 0;
+						std::set<DelEn, DelEnCompare>::const_iterator i(mPotentialDeletions.begin());
+						for(; i != mPotentialDeletions.end(); ++i)
+						{
+							if(i->mSizeInBlocks > mMaxSizeInPotentialDeletions)
+							{
+								mMaxSizeInPotentialDeletions = i->mSizeInBlocks;
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	{
+		// Recurse into subdirectories
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+		{
+			// Next level
+			ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
+			
+			if(!ScanDirectory(en->GetObjectID()))
+			{
+				// Halting operation
+				return false;
+			}
+		}
+	}
+	
+	return true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &, const HousekeepStoreAccount::DelEnd &)
+//		Purpose: Comparison function for set
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y)
+{
+	// STL spec says this:
+	// A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. 
+
+	// The sort order here is intended to preserve the entries of most value, that is, the newest objects
+	// which are on a mark boundary.
+	
+	// Reverse order age, so oldest goes first
+	if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark)
+	{
+		return true;
+	}
+	else if(x.mVersionAgeWithinMark < y.mVersionAgeWithinMark)
+	{
+		return false;
+	}
+
+	// but mark number in ascending order, so that the oldest marks are deleted first
+	if(x.mMarkNumber < y.mMarkNumber)
+	{
+		return true;
+	}
+	else if(x.mMarkNumber > y.mMarkNumber)
+	{
+		return false;
+	}
+
+	// Just compare object ID now to put the oldest objects first
+	return x.mObjectID < y.mObjectID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteFiles()
+//		Purpose: Delete the files targeted for deletion, returning
+//			 true if the operation was interrupted
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteFiles()
+{
+	// Only delete files if the deletion target is greater than zero
+	// (otherwise we delete one file each time round, which gradually deletes the old versions)
+	if(mDeletionSizeTarget <= 0)
+	{
+		// Not interrupted
+		return false;
+	}
+
+	// Iterate through the set of potential deletions, until enough has been deleted.
+	// (there is likely to be more in the set than should be actually deleted).
+	for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i)
+	{
+#ifndef WIN32
+		if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+		{
+			mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+			// Check for having to stop
+			if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+			{
+				// Need to abort now
+				return true;
+			}
+		}
+#endif
+
+		// Load up the directory it's in
+		// Get the filename
+		std::string dirFilename;
+		BackupStoreDirectory dir;
+		int64_t dirSizeInBlocksOrig = 0;
+		{
+			MakeObjectFilename(i->mInDirectory, dirFilename);
+			std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
+			dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks();
+			dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+		}
+		
+		// Delete the file
+		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+		BOX_INFO("Housekeeping removed " <<
+			(i->mIsFlagDeleted ? "deleted" : "old") <<
+			" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
+			" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
+		
+		// Stop if the deletion target has been matched or exceeded
+		// (checking here rather than at the beginning will tend to reduce the
+		// space to slightly less than the soft limit, which will allow the backup
+		// client to start uploading files again)
+		if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget)
+		{
+			break;
+		}
+	}
+
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteFile(int64_t, int64_t,
+//			 BackupStoreDirectory &, const std::string &, int64_t)
+//		Purpose: Delete a file. Takes the directory already loaded
+//			 in and the filename, for efficiency in both the
+//			 usage scenarios.
+//		Created: 15/7/04
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+{
+	// Find the entry inside the directory
+	bool wasDeleted = false;
+	bool wasOldVersion = false;
+	int64_t deletedFileSizeInBlocks = 0;
+	// A pointer to an object which requires committing if the directory save goes OK
+	std::auto_ptr<RaidFileWrite> padjustedEntry;
+	// BLOCK
+	{
+		BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
+		if(pentry == 0)
+		{
+			BOX_ERROR("Housekeeping on account " <<
+				BOX_FORMAT_ACCOUNT(mAccountID) << " "
+				"found error: object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) << " "
+				"not found in dir " << 
+				BOX_FORMAT_OBJECTID(InDirectory) << ", "
+				"indicates logic error/corruption? Run "
+				"bbstoreaccounts check <accid> fix");
+			return;
+		}
+		
+		// Record the flags it's got set
+		wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
+		wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+		// Check this should be deleted
+		if(!wasDeleted && !wasOldVersion)
+		{
+			// Things changed size we were last around
+			return;
+		}
+		
+		// Record size
+		deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
+		
+		// If the entry is involved in a chain of patches, it needs to be handled
+		// a bit more carefully.
+		if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
+		{
+			// This entry is a patch from a newer entry. Just need to update the info on that entry.
+			BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+			if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
+			{
+				THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+			}
+			// Change the info in the newer entry so that this no longer points to this entry
+			pnewer->SetDependsOlder(0);
+		}
+		else if(pentry->GetDependsOlder() != 0)
+		{
+			BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
+			if(pentry->GetDependsNewer() == 0)
+			{
+				// There exists an older version which depends on this one. Need to combine the two over that one.
+
+				// Adjust the other entry in the directory
+				if(polder == 0 || polder->GetDependsNewer() != ObjectID)
+				{
+					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+				}
+				// Change the info in the older entry so that this no longer points to this entry
+				polder->SetDependsNewer(0);
+			}
+			else
+			{
+				// This entry is in the middle of a chain, and two patches need combining.
+				
+				// First, adjust the directory entries
+				BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+				if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
+					|| polder == 0 || polder->GetDependsNewer() != ObjectID)
+				{
+					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+				}
+				// Remove the middle entry from the linked list by simply using the values from this entry
+				pnewer->SetDependsOlder(pentry->GetDependsOlder());
+				polder->SetDependsNewer(pentry->GetDependsNewer());
+			}
+			
+			// COMMON CODE to both cases
+
+			// Generate the filename of the older version
+			std::string objFilenameOlder;
+			MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
+			// Open it twice (it's the diff)
+			std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+			std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+			// Open this file
+			std::string objFilename;
+			MakeObjectFilename(ObjectID, objFilename);
+			std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
+			// And open a write file to overwrite the other directory entry
+			padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
+				objFilenameOlder, mNewRefCounts[ObjectID]));
+			padjustedEntry->Open(true /* allow overwriting */);
+
+			if(pentry->GetDependsNewer() == 0)
+			{
+				// There exists an older version which depends on this one. Need to combine the two over that one.
+				BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
+			}
+			else
+			{
+				// This entry is in the middle of a chain, and two patches need combining.
+				BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
+			}
+			// The file will be committed later when the directory is safely commited.
+			
+			// Work out the adjusted size
+			int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
+			int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
+			mBlocksUsedDelta += sizeDelta;
+			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+			{
+				mBlocksInDeletedFilesDelta += sizeDelta;
+			}
+			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+			{
+				mBlocksInOldFilesDelta += sizeDelta;
+			}
+			polder->SetSizeInBlocks(newSize);
+		}
+		
+		// pentry no longer valid
+	}
+	
+	// Delete it from the directory
+	rDirectory.DeleteEntry(ObjectID);
+	
+	// Save directory back to disc
+	// BLOCK
+	int64_t dirRevisedSize = 0;
+	{
+		RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
+			mNewRefCounts[InDirectory]);
+		writeDir.Open(true /* allow overwriting */);
+		rDirectory.WriteToStream(writeDir);
+
+		// get the disc usage (must do this before commiting it)
+		dirRevisedSize = writeDir.GetDiscUsageInBlocks();
+
+		// Commit directory
+		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+		// adjust usage counts for this directory
+		if(dirRevisedSize > 0)
+		{
+			int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
+			mBlocksUsedDelta += adjust;
+			mBlocksInDirectoriesDelta += adjust;
+		}
+	}
+
+	// Commit any new adjusted entry
+	if(padjustedEntry.get() != 0)
+	{
+		padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+		padjustedEntry.reset();	// delete it now
+	}
+
+	// Drop reference count by one. If it reaches zero, delete the file.
+	if(--mNewRefCounts[ObjectID] == 0)
+	{
+		// Delete from disc
+		BOX_TRACE("Removing unreferenced object " <<
+			BOX_FORMAT_OBJECTID(ObjectID));
+		std::string objFilename;
+		MakeObjectFilename(ObjectID, objFilename);
+		RaidFileWrite del(mStoreDiscSet, objFilename,
+			mNewRefCounts[ObjectID]);
+		del.Delete();
+	}
+	else
+	{
+		BOX_TRACE("Preserving object " <<
+			BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
+			mNewRefCounts[ObjectID] << " references");
+	}
+
+	// Adjust counts for the file
+	++mFilesDeleted;
+	mBlocksUsedDelta -= deletedFileSizeInBlocks;
+	if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+	if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+	
+	// Delete the directory?
+	// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
+	if(rDirectory.GetNumberOfEntries() == 0)
+	{
+		// Candidate for deletion
+		mEmptyDirectories.push_back(InDirectory);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteEmptyDirectories()
+//		Purpose: Remove any empty directories which are also marked as deleted in their containing directory,
+//				 returning true if the opertaion was interrupted
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteEmptyDirectories()
+{
+	while(mEmptyDirectories.size() > 0)
+	{
+		std::vector<int64_t> toExamine;
+
+		// Go through list
+		for(std::vector<int64_t>::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i)
+		{
+#ifndef WIN32
+			if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+			{
+				mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+				// Check for having to stop
+				if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+				{
+					// Need to abort now
+					return true;
+				}
+			}
+#endif
+
+			// Do not delete the root directory
+			if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID)
+			{
+				continue;
+			}
+
+			DeleteEmptyDirectory(*i, toExamine);
+		}
+
+		// Remove contents of empty directories
+		mEmptyDirectories.clear();
+		// Swap in new, so it's examined next time round
+		mEmptyDirectories.swap(toExamine);
+	}
+	
+	// Not interrupted
+	return false;
+}
+
+void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
+	std::vector<int64_t>& rToExamine)
+{
+	// Load up the directory to potentially delete
+	std::string dirFilename;
+	BackupStoreDirectory dir;
+	int64_t dirSizeInBlocks = 0;
+
+	// BLOCK
+	{
+		MakeObjectFilename(dirId, dirFilename);
+		// Check it actually exists (just in case it gets
+		// added twice to the list)
+		if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
+		{
+			// doesn't exist, next!
+			return;
+		}
+		// load
+		std::auto_ptr<RaidFileRead> dirStream(
+			RaidFileRead::Open(mStoreDiscSet, dirFilename));
+		dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+		dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+	}
+
+	// Make sure this directory is actually empty
+	if(dir.GetNumberOfEntries() != 0)
+	{
+		// Not actually empty, try next one
+		return;
+	}
+
+	// Candidate for deletion... open containing directory
+	std::string containingDirFilename;
+	BackupStoreDirectory containingDir;
+	int64_t containingDirSizeInBlocksOrig = 0;
+	{
+		MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
+		std::auto_ptr<RaidFileRead> containingDirStream(
+			RaidFileRead::Open(mStoreDiscSet,
+				containingDirFilename));
+		containingDirSizeInBlocksOrig = 
+			containingDirStream->GetDiscUsageInBlocks();
+		containingDir.ReadFromStream(*containingDirStream,
+			IOStream::TimeOutInfinite);
+	}
+
+	// Find the entry
+	BackupStoreDirectory::Entry *pdirentry = 
+		containingDir.FindEntryByID(dir.GetObjectID());
+	if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
+	{
+		// Should be deleted
+		containingDir.DeleteEntry(dir.GetObjectID());
+
+		// Is the containing dir now a candidate for deletion?
+		if(containingDir.GetNumberOfEntries() == 0)
+		{
+			rToExamine.push_back(containingDir.GetObjectID());
+		}
+
+		// Write revised parent directory
+		RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
+			mNewRefCounts[containingDir.GetObjectID()]);
+		writeDir.Open(true /* allow overwriting */);
+		containingDir.WriteToStream(writeDir);
+
+		// get the disc usage (must do this before commiting it)
+		int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+		// Commit directory
+		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+		// adjust usage counts for this directory
+		if(dirSize > 0)
+		{
+			int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
+			mBlocksUsedDelta += adjust;
+			mBlocksInDirectoriesDelta += adjust;
+		}
+
+		
+		if (--mNewRefCounts[dir.GetObjectID()] == 0)
+		{
+			// Delete the directory itself
+			RaidFileWrite del(mStoreDiscSet, dirFilename,
+				mNewRefCounts[dir.GetObjectID()]);
+			del.Delete();
+		}
+
+		BOX_INFO("Housekeeping removed empty deleted dir " <<
+			BOX_FORMAT_OBJECTID(dirId));
+
+		// And adjust usage counts for the directory that's
+		// just been deleted
+		mBlocksUsedDelta -= dirSizeInBlocks;
+		mBlocksInDirectoriesDelta -= dirSizeInBlocks;
+
+		// Update count
+		++mEmptyDirectoriesDeleted;
+	}
+}
+

Copied: box/trunk/lib/backupstore/HousekeepStoreAccount.h (from rev 3113, box/trunk/bin/bbstored/HousekeepStoreAccount.h)
===================================================================
--- box/trunk/lib/backupstore/HousekeepStoreAccount.h	                        (rev 0)
+++ box/trunk/lib/backupstore/HousekeepStoreAccount.h	2012-06-29 22:15:36 UTC (rev 3114)
@@ -0,0 +1,111 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    HousekeepStoreAccount.h
+//		Purpose: Action class to perform housekeeping on a store account
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef HOUSEKEEPSTOREACCOUNT__H
+#define HOUSEKEEPSTOREACCOUNT__H
+
+#include <string>
+#include <set>
+#include <vector>
+
+class BackupStoreDirectory;
+
+class HousekeepingCallback
+{
+	public:
+	virtual ~HousekeepingCallback() {}
+	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    HousekeepStoreAccount
+//		Purpose: Action class to perform housekeeping on a store account
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+class HousekeepStoreAccount
+{
+public:
+	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot,
+		int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback);
+	~HousekeepStoreAccount();
+	
+	bool DoHousekeeping(bool KeepTryingForever = false);
+	int GetRefCountsAdjusted() { return mRefCountsAdjusted; }
+	
+private:
+	// utility functions
+	void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
+
+	bool ScanDirectory(int64_t ObjectID);
+	bool DeleteFiles();
+	bool DeleteEmptyDirectories();
+	void DeleteEmptyDirectory(int64_t dirId,
+		std::vector<int64_t>& rToExamine);
+	void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
+
+private:
+	typedef struct
+	{
+		int64_t mObjectID;
+		int64_t mInDirectory;
+		int64_t mSizeInBlocks;
+		int32_t mMarkNumber;
+		int32_t mVersionAgeWithinMark;	// 0 == current, 1 latest old version, etc
+		bool    mIsFlagDeleted; // false for files flagged "Old"
+	} DelEn;
+	
+	struct DelEnCompare
+	{
+		bool operator()(const DelEn &x, const DelEn &y);
+	};
+	
+	int mAccountID;
+	std::string mStoreRoot;
+	int mStoreDiscSet;
+	HousekeepingCallback* mpHousekeepingCallback;
+	
+	int64_t mDeletionSizeTarget;
+	
+	std::set<DelEn, DelEnCompare> mPotentialDeletions;
+	int64_t mPotentialDeletionsTotalSize;
+	int64_t mMaxSizeInPotentialDeletions;
+	
+	// List of directories which are empty, and might be good for deleting
+	std::vector<int64_t> mEmptyDirectories;
+	
+	// The re-calculated blocks used stats
+	int64_t mBlocksUsed;
+	int64_t mBlocksInOldFiles;
+	int64_t mBlocksInDeletedFiles;
+	int64_t mBlocksInDirectories;
+
+	// Deltas from deletion
+	int64_t mBlocksUsedDelta;
+	int64_t mBlocksInOldFilesDelta;
+	int64_t mBlocksInDeletedFilesDelta;
+	int64_t mBlocksInDirectoriesDelta;
+	
+	// Deletion count
+	int64_t mFilesDeleted;
+	int64_t mEmptyDirectoriesDeleted;
+
+	// New reference count list
+	std::vector<uint32_t> mNewRefCounts;
+	bool mSuppressRefCountChangeWarnings;
+	int mRefCountsAdjusted;
+	
+	// Poll frequency
+	int mCountUntilNextInterprocessMsgCheck;
+};
+
+#endif // HOUSEKEEPSTOREACCOUNT__H
+



From subversion at boxbackup.org  Fri Jun 29 23:18:36 2012
From: subversion at boxbackup.org (subversion at boxbackup.org)
Date: Fri, 29 Jun 2012 23:18:36 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r3115 - box/trunk/lib/common
Message-ID: <201206292218.q5TMIaZM068336@wm.boxbackup.org>

Author: chris
Date: 2012-06-29 23:18:36 +0100 (Fri, 29 Jun 2012)
New Revision: 3115

Modified:
   box/trunk/lib/common/UnixUser.cpp
   box/trunk/lib/common/UnixUser.h
Log:
Allow UnixUser to be created with a std::string for C++ style.


Modified: box/trunk/lib/common/UnixUser.cpp
===================================================================
--- box/trunk/lib/common/UnixUser.cpp	2012-06-29 22:15:36 UTC (rev 3114)
+++ box/trunk/lib/common/UnixUser.cpp	2012-06-29 22:18:36 UTC (rev 3115)
@@ -31,13 +31,13 @@
 //		Created: 21/1/04
 //
 // --------------------------------------------------------------------------
-UnixUser::UnixUser(const char *Username)
+UnixUser::UnixUser(const std::string& Username)
 	: mUID(0),
 	  mGID(0),
 	  mRevertOnDestruction(false)
 {
 	// Get password info
-	struct passwd *pwd = ::getpwnam(Username);
+	struct passwd *pwd = ::getpwnam(Username.c_str());
 	if(pwd == 0)
 	{
 		THROW_EXCEPTION(CommonException, CouldNotLookUpUsername)

Modified: box/trunk/lib/common/UnixUser.h
===================================================================
--- box/trunk/lib/common/UnixUser.h	2012-06-29 22:15:36 UTC (rev 3114)
+++ box/trunk/lib/common/UnixUser.h	2012-06-29 22:18:36 UTC (rev 3115)
@@ -13,7 +13,7 @@
 class UnixUser
 {
 public:
-	UnixUser(const char *Username);
+	UnixUser(const std::string& Username);
 	UnixUser(uid_t UID, gid_t GID);
 	~UnixUser();
 private:



From subversion at boxbackup.org  Fri Jun 29 23:19:10 2012
From: subversion at boxbackup.org (subversion at boxbackup.org)
Date: Fri, 29 Jun 2012 23:19:10 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r3116 - box/trunk/bin/bbackupd
Message-ID: <201206292219.q5TMJARF068354@wm.boxbackup.org>

Author: chris
Date: 2012-06-29 23:19:10 +0100 (Fri, 29 Jun 2012)
New Revision: 3116

Modified:
   box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp
Log:
Throw an exception if we fail to open inode database, even in release builds.


Modified: box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp	2012-06-29 22:18:36 UTC (rev 3115)
+++ box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp	2012-06-29 22:19:10 UTC (rev 3116)
@@ -116,8 +116,14 @@
 	
 	mpDepot = dpopen(Filename, mode, 0);
 	
-	ASSERT_DBM_OK(mpDepot, "Failed to open inode database", mFilename,
-		BackupStoreException, BerkelyDBFailure);
+	if(!mpDepot)
+	{
+		BOX_WARNING(BOX_DBM_MESSAGE("Failed to open inode "
+			"database: " << mFilename));
+		THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure,
+			BOX_DBM_MESSAGE("Failed to open inode database: " <<
+				mFilename));
+	}
 	
 	// Read only flag
 	mReadOnly = ReadOnly;