From boxbackup-dev at fluffy.co.uk  Sun May  4 16:28:56 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Sun,  4 May 2008 16:28:56 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2156 - box/trunk/lib/backupstore
Message-ID: <20080504152856.E1B8E325004@www.boxbackup.org>

Author: chris
Date: 2008-05-04 16:28:55 +0100 (Sun, 04 May 2008)
New Revision: 2156

Modified:
   box/trunk/lib/backupstore/BackupStoreCheck2.cpp
Log:
Fix typos in comments.


Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp
===================================================================
--- box/trunk/lib/backupstore/BackupStoreCheck2.cpp	2008-04-30 22:57:59 UTC (rev 2155)
+++ box/trunk/lib/backupstore/BackupStoreCheck2.cpp	2008-05-04 15:28:55 UTC (rev 2156)
@@ -399,11 +399,11 @@
 		dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
 	}
 	
-	// Add a new entry in an appropraite place
+	// Add a new entry in an appropriate place
 	dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks,
 		IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
 
-	// Fix any flags which have been broken, which there's a good change of going
+	// Fix any flags which have been broken, which there's a good chance of doing
 	dir.CheckAndFix();
 	
 	// Write it out



From boxbackup-dev at fluffy.co.uk  Mon May  5 15:44:39 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Mon,  5 May 2008 15:44:39 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2157 - in box/trunk: bin/bbackupquery lib/backupclient
Message-ID: <20080505144439.B635F325004@www.boxbackup.org>

Author: chris
Date: 2008-05-05 15:44:38 +0100 (Mon, 05 May 2008)
New Revision: 2157

Modified:
   box/trunk/bin/bbackupquery/BackupQueries.cpp
   box/trunk/bin/bbackupquery/documentation.txt
   box/trunk/lib/backupclient/BackupClientRestore.cpp
   box/trunk/lib/backupclient/BackupClientRestore.h
Log:
Add restore -f option to force restore to continue after an error.


Modified: box/trunk/bin/bbackupquery/BackupQueries.cpp
===================================================================
--- box/trunk/bin/bbackupquery/BackupQueries.cpp	2008-05-04 15:28:55 UTC (rev 2156)
+++ box/trunk/bin/bbackupquery/BackupQueries.cpp	2008-05-05 14:44:38 UTC (rev 2157)
@@ -217,7 +217,7 @@
 		{ "getobject", "" },
 		{ "get",  "i" },
 		{ "compare", "alcqAEQ" },
-		{ "restore", "dri" },
+		{ "restore", "drif" },
 		{ "help", "" },
 		{ "usage", "" },
 		{ "undelete", "" },
@@ -1966,7 +1966,7 @@
 	// Check arguments
 	if(args.size() != 2)
 	{
-		BOX_ERROR("Incorrect usage. restore [-d] [-r] [-i] <remote-name> <local-name>");
+		BOX_ERROR("Incorrect usage. restore [-drif] <remote-name> <local-name>");
 		return;
 	}
 
@@ -2029,7 +2029,8 @@
 			localName.c_str(), 
 			true /* print progress dots */, restoreDeleted, 
 			false /* don't undelete after restore! */, 
-			opts['r'] /* resume? */);
+			opts['r'] /* resume? */,
+			opts['f'] /* force continue after errors */);
 	}
 	catch(std::exception &e)
 	{
@@ -2050,13 +2051,20 @@
 		BOX_INFO("Restore complete.");
 		break;
 	
+	case Restore_CompleteWithErrors:
+		BOX_WARNING("Restore complete, but some files could not be "
+			"restored.");
+		break;
+	
 	case Restore_ResumePossible:
-		BOX_ERROR("Resume possible -- repeat command with -r flag to resume");
+		BOX_ERROR("Resume possible -- repeat command with -r flag "
+			"to resume.");
 		SetReturnCode(COMMAND_RETURN_ERROR);
 		break;
 	
 	case Restore_TargetExists:
-		BOX_ERROR("The target directory exists. You cannot restore over an existing directory.");
+		BOX_ERROR("The target directory exists. You cannot restore "
+			"over an existing directory.");
 		SetReturnCode(COMMAND_RETURN_ERROR);
 		break;
 		

Modified: box/trunk/bin/bbackupquery/documentation.txt
===================================================================
--- box/trunk/bin/bbackupquery/documentation.txt	2008-05-04 15:28:55 UTC (rev 2156)
+++ box/trunk/bin/bbackupquery/documentation.txt	2008-05-05 14:44:38 UTC (rev 2157)
@@ -116,7 +116,7 @@
 	This can be used for automated tests.
 <
 
-> restore [-d] [-r] [-i] <directory-name> <local-directory-name>
+> restore [-drif] <directory-name> <local-directory-name>
 
 	Restores a directory to the local disc. The local directory specified
 	must not exist (unless a previous restore is being restarted).
@@ -126,6 +126,7 @@
 	-d -- restore a deleted directory or deleted files inside
 	-r -- resume an interrupted restoration
 	-i -- directory name is actually an ID
+	-f -- force restore to continue if errors are encountered
 
 	If a restore operation is interrupted for any reason, it can be restarted
 	using the -r switch. Restore progress information is saved in a file at

Modified: box/trunk/lib/backupclient/BackupClientRestore.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupClientRestore.cpp	2008-05-04 15:28:55 UTC (rev 2156)
+++ box/trunk/lib/backupclient/BackupClientRestore.cpp	2008-05-05 14:44:38 UTC (rev 2157)
@@ -193,6 +193,8 @@
 {
 	bool PrintDots;
 	bool RestoreDeleted;
+	bool ContinueAfterErrors;
+	bool ContinuedAfterError;
 	std::string mRestoreResumeInfoFilename;
 	RestoreResumeInfo mResumeInfo;
 } RestoreParams;
@@ -387,7 +389,15 @@
 	{
 		BOX_LOG_SYS_ERROR("Failed to create directory '" <<
 			rLocalDirectoryName << "'");
-		return Restore_UnknownError;
+
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 
 	// Save the restore info, in case it's needed later
@@ -400,14 +410,30 @@
 		BOX_ERROR("Failed to save resume info file '" <<
 			Params.mRestoreResumeInfoFilename << "': " <<
 			e.what());
-		return Restore_UnknownError;
+
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 	catch(...)
 	{
 		BOX_ERROR("Failed to save resume info file '" <<
 			Params.mRestoreResumeInfoFilename <<
 			"': unknown error");
-		return Restore_UnknownError;
+
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 
 	// Fetch the directory listing from the server -- getting a
@@ -435,13 +461,29 @@
 	{
 		BOX_ERROR("Failed to restore attributes for '" <<
 			rLocalDirectoryName << "': " << e.what());
-		return Restore_UnknownError;
+
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 	catch(...)
 	{
 		BOX_ERROR("Failed to restore attributes for '" <<
 			rLocalDirectoryName << "': unknown error");
-		return Restore_UnknownError;
+
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 
 	int64_t bytesWrittenSinceLastRestoreInfoSave = 0;
@@ -473,7 +515,15 @@
 					BOX_LOG_SYS_ERROR("Failed to delete "
 						"file '" << localFilename << 
 						"'");
-					return Restore_UnknownError;
+
+					if (Params.ContinueAfterErrors)
+					{
+						Params.ContinuedAfterError = true;
+					}
+					else
+					{
+						return Restore_UnknownError;
+					}
 				}
 				
 				// Request it from the store
@@ -507,14 +557,30 @@
 					BOX_ERROR("Failed to restore file '" <<
 						localFilename << "': " <<
 						e.what());
-					return Restore_UnknownError;
+
+					if (Params.ContinueAfterErrors)
+					{
+						Params.ContinuedAfterError = true;
+					}
+					else
+					{
+						return Restore_UnknownError;
+					}
 				}
 				catch(...)
 				{
 					BOX_ERROR("Failed to restore file '" <<
 						localFilename <<
 						"': unknown error");
-					return Restore_UnknownError;
+
+					if (Params.ContinueAfterErrors)
+					{
+						Params.ContinuedAfterError = true;
+					}
+					else
+					{
+						return Restore_UnknownError;
+					}
 				}
 				
 				// Progress display?
@@ -545,7 +611,15 @@
 						"whether file exists: '" <<
 						localFilename << "': " <<
 						e.what());
-					return Restore_UnknownError;
+
+					if (Params.ContinueAfterErrors)
+					{
+						Params.ContinuedAfterError = true;
+					}
+					else
+					{
+						return Restore_UnknownError;
+					}
 				}
 				catch(...)
 				{
@@ -553,7 +627,15 @@
 						"whether file exists: '" <<
 						localFilename << "': "
 						"unknown error");
-					return Restore_UnknownError;
+
+					if (Params.ContinueAfterErrors)
+					{
+						Params.ContinuedAfterError = true;
+					}
+					else
+					{
+						return Restore_UnknownError;
+					}
 				}
 
 				if(exists)
@@ -606,14 +688,28 @@
 			BOX_ERROR("Failed to save resume info file '" <<
 				Params.mRestoreResumeInfoFilename << "': " <<
 				e.what());
-			return Restore_UnknownError;
+			if (Params.ContinueAfterErrors)
+			{
+				Params.ContinuedAfterError = true;
+			}
+			else
+			{
+				return Restore_UnknownError;
+			}
 		}
 		catch(...)
 		{
 			BOX_ERROR("Failed to save resume info file '" <<
 				Params.mRestoreResumeInfoFilename <<
 				"': unknown error");
-			return Restore_UnknownError;
+			if (Params.ContinueAfterErrors)
+			{
+				Params.ContinuedAfterError = true;
+			}
+			else
+			{
+				return Restore_UnknownError;
+			}
 		}
 		
 		bytesWrittenSinceLastRestoreInfoSave = 0;
@@ -670,13 +766,27 @@
 	{
 		BOX_ERROR("Failed to restore attributes for '" <<
 			rLocalDirectoryName << "': " << e.what());
-		return Restore_UnknownError;
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 	catch(...)
 	{
 		BOX_ERROR("Failed to restore attributes for '" <<
 			rLocalDirectoryName << "': unknown error");
-		return Restore_UnknownError;
+		if (Params.ContinueAfterErrors)
+		{
+			Params.ContinuedAfterError = true;
+		}
+		else
+		{
+			return Restore_UnknownError;
+		}
 	}
 
 	return Restore_Complete;
@@ -687,7 +797,7 @@
 //
 // Function
 //		Name:    BackupClientRestore(BackupProtocolClient &, int64_t,
-//			 const char *, bool, bool)
+//			 const char *, bool, bool, bool, bool, bool)
 //		Purpose: Restore a directory on the server to a local
 //			 directory on the disc. The local directory must not
 //			 already exist.
@@ -707,19 +817,24 @@
 //			 (Won't attempt to overwrite things.)
 //
 //			 Returns Restore_Complete on success. (Exceptions
-//			 on error.)
+//			 on error, unless ContinueAfterError is true and
+//			 the error is recoverable, in which case it returns
+//			 Restore_CompleteWithErrors)
 //		Created: 23/11/03
 //
 // --------------------------------------------------------------------------
 int BackupClientRestore(BackupProtocolClient &rConnection,
 	int64_t DirectoryID, const char *LocalDirectoryName,
 	bool PrintDots, bool RestoreDeleted,
-	bool UndeleteAfterRestoreDeleted, bool Resume)
+	bool UndeleteAfterRestoreDeleted, bool Resume,
+	bool ContinueAfterErrors)
 {
 	// Parameter block
 	RestoreParams params;
 	params.PrintDots = PrintDots;
 	params.RestoreDeleted = RestoreDeleted;
+	params.ContinueAfterErrors = ContinueAfterErrors;
+	params.ContinuedAfterError = false;
 	params.mRestoreResumeInfoFilename = LocalDirectoryName;
 	params.mRestoreResumeInfoFilename += ".boxbackupresume";
 
@@ -782,6 +897,7 @@
 	// Delete the resume information file
 	::unlink(params.mRestoreResumeInfoFilename.c_str());
 
-	return Restore_Complete;
+	return params.ContinuedAfterError ? Restore_CompleteWithErrors
+		: Restore_Complete;
 }
 

Modified: box/trunk/lib/backupclient/BackupClientRestore.h
===================================================================
--- box/trunk/lib/backupclient/BackupClientRestore.h	2008-05-04 15:28:55 UTC (rev 2156)
+++ box/trunk/lib/backupclient/BackupClientRestore.h	2008-05-05 14:44:38 UTC (rev 2157)
@@ -15,14 +15,21 @@
 enum
 {
 	Restore_Complete = 0,
-	Restore_ResumePossible = 1,
-	Restore_TargetExists = 2,
-	Restore_TargetPathNotFound = 3,
-	Restore_UnknownError = 4,
+	Restore_ResumePossible,
+	Restore_TargetExists,
+	Restore_TargetPathNotFound,
+	Restore_UnknownError,
+	Restore_CompleteWithErrors,
 };
 
-int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
-	bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false);
+int BackupClientRestore(BackupProtocolClient &rConnection,
+	int64_t DirectoryID,
+	const char *LocalDirectoryName,
+	bool PrintDots = false,
+	bool RestoreDeleted = false,
+	bool UndeleteAfterRestoreDeleted = false,
+	bool Resume = false,
+	bool ContinueAfterErrors = false);
 
 #endif // BACKUPSCLIENTRESTORE__H
 



From boxbackup-dev at fluffy.co.uk  Mon May  5 22:04:34 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Mon,  5 May 2008 22:04:34 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2158 - box/trunk/bin/bbackupquery
Message-ID: <20080505210434.1F174325004@www.boxbackup.org>

Author: chris
Date: 2008-05-05 22:04:33 +0100 (Mon, 05 May 2008)
New Revision: 2158

Modified:
   box/trunk/bin/bbackupquery/BackupQueries.cpp
Log:
Catch server errors when listing a directory.


Modified: box/trunk/bin/bbackupquery/BackupQueries.cpp
===================================================================
--- box/trunk/bin/bbackupquery/BackupQueries.cpp	2008-05-05 14:44:38 UTC (rev 2157)
+++ box/trunk/bin/bbackupquery/BackupQueries.cpp	2008-05-05 21:04:33 UTC (rev 2158)
@@ -410,6 +410,7 @@
 		{
 			BOX_ERROR("Directory '" << args[0] << "' not found "
 				"on store.");
+			SetReturnCode(COMMAND_RETURN_ERROR);
 			return;
 		}
 	}
@@ -435,12 +436,29 @@
 	if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted;
 
 	// Do communication
-	mrConnection.QueryListDirectory(
-			DirID,
-			BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,	// both files and directories
-			excludeFlags,
-			true /* want attributes */);
+	try
+	{
+		mrConnection.QueryListDirectory(
+				DirID,
+				BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+				// both files and directories
+				excludeFlags,
+				true /* want attributes */);
+	}
+	catch (std::exception &e)
+	{
+		BOX_ERROR("Failed to list directory: " << e.what());
+		SetReturnCode(COMMAND_RETURN_ERROR);
+		return;
+	}
+	catch (...)
+	{
+		BOX_ERROR("Failed to list directory: unknown error");
+		SetReturnCode(COMMAND_RETURN_ERROR);
+		return;
+	}
 
+
 	// Retrieve the directory from the stream following
 	BackupStoreDirectory dir;
 	std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream());
@@ -755,6 +773,7 @@
 	if(args.size() != 1 || args[0].size() == 0)
 	{
 		BOX_ERROR("Incorrect usage. cd [-o] [-d] <directory>");
+		SetReturnCode(COMMAND_RETURN_ERROR);
 		return;
 	}
 
@@ -772,6 +791,7 @@
 	if(id == 0)
 	{
 		BOX_ERROR("Directory '" << args[0] << "' not found.");
+		SetReturnCode(COMMAND_RETURN_ERROR);
 		return;
 	}
 	



From boxbackup-dev at fluffy.co.uk  Tue May  6 22:48:53 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Tue,  6 May 2008 22:48:53 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2159 - box/trunk/lib/backupstore
Message-ID: <20080506214853.36876325004@www.boxbackup.org>

Author: chris
Date: 2008-05-06 22:48:52 +0100 (Tue, 06 May 2008)
New Revision: 2159

Modified:
   box/trunk/lib/backupstore/BackupStoreCheck2.cpp
Log:
Fix O(n^2) algorithm to insert lost objects into a directory, which would
become very slow with large directories (e.g. 100,000 files unattached)
due to repeated reading and writing of the directory.


Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp
===================================================================
--- box/trunk/lib/backupstore/BackupStoreCheck2.cpp	2008-05-05 21:04:33 UTC (rev 2158)
+++ box/trunk/lib/backupstore/BackupStoreCheck2.cpp	2008-05-06 21:48:52 UTC (rev 2159)
@@ -95,7 +95,22 @@
 	mBlocksInDirectories += size;
 }
 
+class BackupStoreDirectoryFixer
+{
+	private:
+	BackupStoreDirectory mDirectory;
+	std::string mFilename;
+	std::string mStoreRoot;
+	int mDiscSetNumber;
 
+	public:
+	BackupStoreDirectoryFixer(std::string storeRoot, int discSetNumber,
+		int64_t ID);
+	void InsertObject(int64_t ObjectID, bool IsDirectory,
+		int32_t lostDirNameSerial);
+	~BackupStoreDirectoryFixer();
+};
+
 // --------------------------------------------------------------------------
 //
 // Function
@@ -106,6 +121,10 @@
 // --------------------------------------------------------------------------
 void BackupStoreCheck::CheckUnattachedObjects()
 {
+	typedef std::map<int64_t, BackupStoreDirectoryFixer*> fixers_t;
+	typedef std::pair<int64_t, BackupStoreDirectoryFixer*> fixer_pair_t;
+	fixers_t fixers;
+
 	// Scan all objects, finding ones which have no container
 	for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
 	{
@@ -118,7 +137,9 @@
 			if((flags & Flags_IsContained) == 0)
 			{
 				// Unattached object...
-				BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached.");
+				BOX_WARNING("Object " <<
+					BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
+					" is unattached.");
 				++mNumberErrorsFound;
 
 				// What's to be done?
@@ -196,15 +217,51 @@
 				}
 				ASSERT(putIntoDirectoryID != 0);
 
+				if (!mFixErrors)
+				{
+					continue;
+				}
+
+				BackupStoreDirectoryFixer* pFixer;
+				fixers_t::iterator fi = 
+					fixers.find(putIntoDirectoryID);
+				if (fi == fixers.end())
+				{
+					// no match, create a new one
+					pFixer = new BackupStoreDirectoryFixer(
+						mStoreRoot, mDiscSetNumber,
+						putIntoDirectoryID);
+					fixers.insert(fixer_pair_t(
+						putIntoDirectoryID, pFixer));
+				}
+				else
+				{
+					pFixer = fi->second;
+				}
+
+				int32_t lostDirNameSerial = 0;
+
+				if(flags & Flags_IsDir)
+				{
+					lostDirNameSerial = mLostDirNameSerial++;
+				}
+
 				// Add it to the directory
-				InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID,
-					((flags & Flags_IsDir) == Flags_IsDir));
+				pFixer->InsertObject(pblock->mID[e],
+					((flags & Flags_IsDir) == Flags_IsDir),
+					lostDirNameSerial);
 			}
 		}
 	}
+
+	// clean up all the fixers. Deleting them commits them automatically.
+	for (fixers_t::iterator i = fixers.begin(); i != fixers.end(); i++)
+	{
+		BackupStoreDirectoryFixer* pFixer = i->second;
+		delete pFixer;
+	}
 }
 
-
 // --------------------------------------------------------------------------
 //
 // Function
@@ -261,7 +318,87 @@
 	return true;
 }
 
+BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot,
+	int discSetNumber, int64_t ID)
+: mStoreRoot(storeRoot),
+  mDiscSetNumber(discSetNumber)
+{
+	// Generate filename
+	StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber,
+		mFilename, false /* don't make sure the dir exists */);
+	
+	// Read it in
+	std::auto_ptr<RaidFileRead> file(
+		RaidFileRead::Open(mDiscSetNumber, mFilename));
+	mDirectory.ReadFromStream(*file, IOStream::TimeOutInfinite);
+}
 
+void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory,
+	int32_t lostDirNameSerial)
+{
+	// Data for the object
+	BackupStoreFilename objectStoreFilename;
+	int64_t modTime = 100;	// something which isn't zero or a special time
+	int32_t sizeInBlocks = 0; // suitable for directories
+
+	if(IsDirectory)
+	{
+		// Directory -- simply generate a name for it.
+		char name[32];
+		::sprintf(name, "dir%08x", lostDirNameSerial);
+		objectStoreFilename.SetAsClearFilename(name);
+	}
+	else
+	{
+		// Files require a little more work...
+		// Open file
+		std::string fileFilename;
+		StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot,
+			mDiscSetNumber, fileFilename,
+			false /* don't make sure the dir exists */);
+		std::auto_ptr<RaidFileRead> file(
+			RaidFileRead::Open(mDiscSetNumber, fileFilename));
+
+		// Fill in size information
+		sizeInBlocks = file->GetDiscUsageInBlocks();
+
+		// Read in header
+		file_StreamFormat hdr;
+		if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) ||
+			(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+			&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif		
+			))
+		{
+			// This should never happen, everything has been
+			// checked before.
+			THROW_EXCEPTION(BackupStoreException, Internal)
+		}
+		// This tells us nice things
+		modTime = box_ntoh64(hdr.mModificationTime);
+		// And the filename comes next
+		objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite);
+	}
+
+	// Add a new entry in an appropriate place
+	mDirectory.AddUnattactedObject(objectStoreFilename, modTime,
+		ObjectID, sizeInBlocks,
+		IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
+}
+
+BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer()
+{
+	// Fix any flags which have been broken, which there's a good chance of doing
+	mDirectory.CheckAndFix();
+	
+	// Write it out
+	RaidFileWrite root(mDiscSetNumber, mFilename);
+	root.Open(true /* allow overwriting */);
+	mDirectory.WriteToStream(root);
+	root.Commit(true /* convert to raid now */);
+}
+
 // --------------------------------------------------------------------------
 //
 // Function



From boxbackup-dev at fluffy.co.uk  Tue May  6 23:12:44 2008
From: boxbackup-dev at fluffy.co.uk (Box Backup)
Date: Tue, 06 May 2008 22:12:44 -0000
Subject: [Box Backup-commit] #46: bbackupd only ever saves reverse diffs,
 corrupted files on store may be unrecoverable
Message-ID: <042.6f9c7a176405b4529cb5a42568a7e44f@fluffy.co.uk>

#46: bbackupd only ever saves reverse diffs, corrupted files on store may be
unrecoverable
----------------------------------------------+-----------------------------
 Reporter:  chris                             |       Owner:  chris
     Type:  defect                            |      Status:  new  
 Priority:  normal                            |   Milestone:  0.12 
Component:  bbackupd                          |     Version:  trunk
 Keywords:  bbackupd reverse diff corruption  |  
----------------------------------------------+-----------------------------
 If the last version of a file on the store is corrupted, as reported by
 Magnus Homann, then the reverse diffs are all useless.

 We should (add the option to) save a full copy of the file every so often,
 e.g. every 20 or 100 diffs, to allow recovery of some data in this case,
 at the expense of using more space on the store.

-- 
Ticket URL: <https://www.boxbackup.org/trac/ticket/46>
Box Backup <http://www.boxbackup.org/>
An open source, completely automatic on-line backup system for UNIX.


From boxbackup-dev at fluffy.co.uk  Tue May  6 23:14:27 2008
From: boxbackup-dev at fluffy.co.uk (Box Backup)
Date: Tue, 06 May 2008 22:14:27 -0000
Subject: [Box Backup-commit] Re: #8: Improve handling of directories with many files
In-Reply-To: <043.de184c2e65bf05d14effd4157a8d316b@fluffy.co.uk>
References: <043.de184c2e65bf05d14effd4157a8d316b@fluffy.co.uk>
Message-ID: <052.23a5653d62121c3a5b651a8c6be6be34@fluffy.co.uk>

#8: Improve handling of directories with many files
----------------------------+-----------------------------------------------
  Reporter:  martin         |       Owner:  chris
      Type:  task           |      Status:  new  
  Priority:  normal         |   Milestone:  0.12 
 Component:  box libraries  |     Version:  trunk
Resolution:                 |    Keywords:       
----------------------------+-----------------------------------------------
Changes (by chris):

  * owner:  => chris
  * version:  0.10 => trunk
  * milestone:  0.20 => 0.12

Comment:

 bbackupd gets slower and slower when backing up a directory with many
 files. The problem appears to be that the directory is rewritten after
 each file is added, which is O(n^2) in number of files in the directory.

-- 
Ticket URL: <https://www.boxbackup.org/trac/ticket/8#comment:2>
Box Backup <http://www.boxbackup.org/>
An open source, completely automatic on-line backup system for UNIX.


From boxbackup-dev at fluffy.co.uk  Tue May  6 23:20:11 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Tue,  6 May 2008 23:20:11 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2160 - box/trunk/lib/backupstore
Message-ID: <20080506222011.3E6F1325009@www.boxbackup.org>

Author: chris
Date: 2008-05-06 23:20:10 +0100 (Tue, 06 May 2008)
New Revision: 2160

Modified:
   box/trunk/lib/backupstore/BackupStoreCheck2.cpp
Log:
Remove obsolete BackupStoreCheck::InsertObjectIntoDirectory class,
replaced with BackupStoreDirectoryFixer.


Modified: box/trunk/lib/backupstore/BackupStoreCheck2.cpp
===================================================================
--- box/trunk/lib/backupstore/BackupStoreCheck2.cpp	2008-05-06 21:48:52 UTC (rev 2159)
+++ box/trunk/lib/backupstore/BackupStoreCheck2.cpp	2008-05-06 22:20:10 UTC (rev 2160)
@@ -472,91 +472,6 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool)
-//		Purpose: 
-//		Created: 22/4/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory)
-{
-	if(!mFixErrors)
-	{
-		// Don't do anything if we're not supposed to fix errors
-		return;
-	}
-
-	// Data for the object
-	BackupStoreFilename objectStoreFilename;
-	int64_t modTime = 100;	// something which isn't zero or a special time
-	int32_t sizeInBlocks = 0; // suitable for directories
-
-	if(IsDirectory)
-	{
-		// Directory -- simply generate a name for it.
-		char name[32];
-		::sprintf(name, "dir%08x", mLostDirNameSerial++);
-		objectStoreFilename.SetAsClearFilename(name);
-	}
-	else
-	{
-		// Files require a little more work...
-		// Open file
-		std::string fileFilename;
-		StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */);
-		std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename));
-		// Fill in size information
-		sizeInBlocks = file->GetDiscUsageInBlocks();
-		// Read in header
-		file_StreamFormat hdr;
-		if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
-			&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
-#endif		
-			))
-		{
-			// This should never happen, everything has been checked before.
-			THROW_EXCEPTION(BackupStoreException, Internal)
-		}
-		// This tells us nice things
-		modTime = box_ntoh64(hdr.mModificationTime);
-		// And the filename comes next
-		objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite);
-	}
-
-	// Directory object
-	BackupStoreDirectory dir;
-
-	// Generate filename
-	std::string filename;
-	StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
-	
-	// Read it in
-	{
-		std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
-		dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
-	}
-	
-	// Add a new entry in an appropriate place
-	dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks,
-		IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
-
-	// Fix any flags which have been broken, which there's a good chance of doing
-	dir.CheckAndFix();
-	
-	// Write it out
-	if(mFixErrors)
-	{
-		RaidFileWrite root(mDiscSetNumber, filename);
-		root.Open(true /* allow overwriting */);
-		dir.WriteToStream(root);
-		root.Commit(true /* convert to raid now */);
-	}
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
 //		Name:    BackupStoreCheck::FixDirsWithWrongContainerID()
 //		Purpose: Rewrites container IDs where required
 //		Created: 22/4/04



From boxbackup-dev at fluffy.co.uk  Thu May 15 19:14:47 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Thu, 15 May 2008 19:14:47 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2161 - box/boxbackup-web
Message-ID: <20080515181447.EA28C325007@www.boxbackup.org>

Author: jamesog
Date: 2008-05-15 19:14:44 +0100 (Thu, 15 May 2008)
New Revision: 2161

Added:
   box/boxbackup-web/security.html
Modified:
   box/boxbackup-web/bbstyles.css
   box/boxbackup-web/index.html
Log:
Add a note about the Debian OpenSSL vulnerability.


Modified: box/boxbackup-web/bbstyles.css
===================================================================
--- box/boxbackup-web/bbstyles.css	2008-05-06 22:20:10 UTC (rev 2160)
+++ box/boxbackup-web/bbstyles.css	2008-05-15 18:14:44 UTC (rev 2161)
@@ -27,7 +27,13 @@
 	margin-left: 250px;
 	position: relative;
 	width: auto }
-	
+
+#security-announce {
+	border: 1px solid #c00;
+	color: #c00;
+	padding: 5px;
+}
+
 tr,td {font-size: 1em;
 	line-height: 150%;
 	text-align: left;

Modified: box/boxbackup-web/index.html
===================================================================
--- box/boxbackup-web/index.html	2008-05-06 22:20:10 UTC (rev 2160)
+++ box/boxbackup-web/index.html	2008-05-15 18:14:44 UTC (rev 2161)
@@ -16,6 +16,8 @@
 
 <h1>Box Backup</h1>
 
+<p id="security-announce"><strong>SECURITY ANNOUNCEMENT:</strong> Please click <a href="security.html">here</a> to read more on the recent OpenSSL vulnerability in Debian GNU/Linux and how it affects Box Backup. <em>[2007/05/15]</p>
+
 <p>An open source, completely automatic on-line backup system for UNIX.
 <ul>
 <li>All backed up data is stored on the server in files on a filesystem -- no tape or archive devices are used

Added: box/boxbackup-web/security.html
===================================================================
--- box/boxbackup-web/security.html	                        (rev 0)
+++ box/boxbackup-web/security.html	2008-05-15 18:14:44 UTC (rev 2161)
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="content-type" content="text/html;charset=iso-8859-1" />
+<title>Box Backup Security Announcement</title>
+<link rel="stylesheet" href="bbstyles.css" type="text/css" />
+</head>
+<body>
+<div align="center">
+<div id="header">
+<div id="logo">
+<img src="images/bblogo.png" alt="logo" height="65" width="331" border="0" vspace="5" align="middle" /> <img src="images/stepahead.png" alt="a step ahead in data security" width="182" height="11" hspace="10" vspace="20" border="0" align="middle" /></div>
+</div>
+<div id="page">
+
+<h1>Security Announcement</h1>
+
+<p>On 13th May, 2008, the Debian Project announced a vulnerability in their OpenSSL package. See <a href="http://lists.debian.org/debian-security-announce/2008/msg00152.html">the announcement</a> for more information.</p>
+
+<p>This page attempts to explain how this may affect users of Box Backup using Debian systems.</p>
+
+<p>First and foremost, Box Backup users who generated their certificates/keys on affected Debian systems should consider the security of their backups compromised. The server admin or anyone able to deduce the private key of a server or client certificate could have read your data.</p>
+
+<p>If the <acronym title="Pseudo Random Number Generator">PRNG</acronym> in your OpenSSL was insufficiently random, you need to:
+	<ul>
+		<li>Regenerate all affected certificates, which have been generated or signed on an affected system</li>
+		<li>Regenerate all the data keys (*-FileEncKeys.raw)</li>
+		<li>Destroy the data stored on your server to an appropriate level of security (overwrite with zeros at the least, more if you're paranoid)</li>
+		<li>Upload everything again</li>
+		<li>Take appropriate measures under the assumption that you have been storing your data in plain text on a public server without authentication.</li>
+	</ul>
+(i.e. start from scratch, destroying all trace of the backed up data, and take other measures to mitigate the exposure of your secrets.)</p>
+
+<p>You need only worry about the systems where:
+	<ul>
+		<li>The certificates were generated or signed</li>
+		<li>The .raw keys were generated</li>
+		<li>The client which backed up data</li>
+	</ul>
+</p>
+
+<p>If your server has this flaw, but no key material or signing was done on it, you should be fine.</p>
+
+<p>If your certificates are weak but the .raw keys are fine, assume that your data has not been read, but that an attacker logged in and corrupted your backups. Destroy the data and start again.</p>
+
+<p>If your certificates are fine<sup>[1]</sup> but a client's .raw file isn't <em>OR</em> an affected client backed up data, just destroy data for that client and restart with that client. Assume that client's data has been exposed to the server admin, but not the outside world.</p>
+
+<p><sup>[1]</sup> Meaning that all cryptographic operations were done on an unaffected machine, including the generation of the client certificate keys before signing elsewhere.</p>
+
+</div>
+</div>
+</body>
+</html>



From boxbackup-dev at fluffy.co.uk  Thu May 15 19:23:36 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Thu, 15 May 2008 19:23:36 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2162 - box/boxbackup-web
Message-ID: <20080515182336.31BD8325007@www.boxbackup.org>

Author: jamesog
Date: 2008-05-15 19:23:36 +0100 (Thu, 15 May 2008)
New Revision: 2162

Modified:
   box/boxbackup-web/index.html
Log:
Missed an </em>.


Modified: box/boxbackup-web/index.html
===================================================================
--- box/boxbackup-web/index.html	2008-05-15 18:14:44 UTC (rev 2161)
+++ box/boxbackup-web/index.html	2008-05-15 18:23:36 UTC (rev 2162)
@@ -16,7 +16,7 @@
 
 <h1>Box Backup</h1>
 
-<p id="security-announce"><strong>SECURITY ANNOUNCEMENT:</strong> Please click <a href="security.html">here</a> to read more on the recent OpenSSL vulnerability in Debian GNU/Linux and how it affects Box Backup. <em>[2007/05/15]</p>
+<p id="security-announce"><strong>SECURITY ANNOUNCEMENT:</strong> Please click <a href="security.html">here</a> to read more on the recent OpenSSL vulnerability in Debian GNU/Linux and how it affects Box Backup. <em>[2007/05/15]</em></p>
 
 <p>An open source, completely automatic on-line backup system for UNIX.
 <ul>



From boxbackup-dev at fluffy.co.uk  Wed May 28 10:34:54 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 10:34:54 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2163 - box/trunk/infrastructure
Message-ID: <20080528093454.A03C9325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 10:34:53 +0100 (Wed, 28 May 2008)
New Revision: 2163

Modified:
   box/trunk/infrastructure/makebuildenv.pl.in
Log:
Define Box module name on the compiler command line (useful for tests 
self-reporting and setting logger tags).

Add the "c" option to "ar" to suppress warning messages about creating 
archives.


Modified: box/trunk/infrastructure/makebuildenv.pl.in
===================================================================
--- box/trunk/infrastructure/makebuildenv.pl.in	2008-05-15 18:23:36 UTC (rev 2162)
+++ box/trunk/infrastructure/makebuildenv.pl.in	2008-05-28 09:34:53 UTC (rev 2163)
@@ -683,6 +683,7 @@
 		if ($is_cpp)
 		{
 			$make .= "\t\$(_CXX) \$(CXXFLAGS) $compile_line_extra ".
+				"-DBOX_MODULE=\"\\\"$mod\\\"\" " .
 				"-c $file -o $out_name\n\n";
 		}
 		elsif ($is_rc)
@@ -771,7 +772,7 @@
 	{
 		# make a library archive...
 		print MAKE "\t\$(HIDE) (echo -n > $end_target; rm $end_target)\n";
-		print MAKE "\t\$(_AR) -q $end_target $o_file_list\n";
+		print MAKE "\t\$(_AR) cq $end_target $o_file_list\n";
 		print MAKE "\t\$(_RANLIB) $end_target\n";
 	}
 	else



From boxbackup-dev at fluffy.co.uk  Wed May 28 10:38:13 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 10:38:13 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2164 - in box/trunk: . test/bbackupd/testfiles
Message-ID: <20080528093813.10DBD325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 10:38:12 +0100 (Wed, 28 May 2008)
New Revision: 2164

Added:
   box/trunk/test/bbackupd/testfiles/bbackupd-exclude.conf.in
   box/trunk/test/bbackupd/testfiles/bbackupd-snapshot.conf.in
   box/trunk/test/bbackupd/testfiles/bbackupd-symlink.conf.in
Modified:
   box/trunk/configure.ac
Log:
Add new bbackupd config files for bbackupd test.


Modified: box/trunk/configure.ac
===================================================================
--- box/trunk/configure.ac	2008-05-28 09:34:53 UTC (rev 2163)
+++ box/trunk/configure.ac	2008-05-28 09:38:12 UTC (rev 2164)
@@ -316,6 +316,9 @@
                    runtest.pl
                    test/backupstorefix/testfiles/testbackupstorefix.pl
                    test/bbackupd/testfiles/bbackupd.conf
+                   test/bbackupd/testfiles/bbackupd-exclude.conf
+                   test/bbackupd/testfiles/bbackupd-snapshot.conf
+                   test/bbackupd/testfiles/bbackupd-symlink.conf
                    test/bbackupd/testfiles/extcheck1.pl
                    test/bbackupd/testfiles/extcheck2.pl
                    test/bbackupd/testfiles/notifyscript.pl

Copied: box/trunk/test/bbackupd/testfiles/bbackupd-exclude.conf.in (from rev 2078, box/trunk/test/bbackupd/testfiles/bbackupd.conf.in)
===================================================================
--- box/trunk/test/bbackupd/testfiles/bbackupd-exclude.conf.in	                        (rev 0)
+++ box/trunk/test/bbackupd/testfiles/bbackupd-exclude.conf.in	2008-05-28 09:38:12 UTC (rev 2164)
@@ -0,0 +1,47 @@
+
+CertificateFile = testfiles/clientCerts.pem
+PrivateKeyFile = testfiles/clientPrivKey.pem
+TrustedCAsFile = testfiles/clientTrustedCAs.pem
+
+KeysFile = testfiles/bbackupd.keys
+
+DataDirectory = testfiles/bbackupd-data
+
+StoreHostname = localhost
+StorePort = 22011
+AccountNumber = 0x01234567
+
+UpdateStoreInterval = 3
+MinimumFileAge = 4
+MaxUploadWait = 24
+DeleteRedundantLocationsAfter = 10
+
+FileTrackingSizeThreshold = 1024
+DiffingUploadSizeThreshold = 1024
+
+MaximumDiffingTime = 3
+KeepAliveTime = 1
+
+ExtendedLogging = no
+ExtendedLogFile = testfiles/bbackupd.log
+
+CommandSocket = testfiles/bbackupd.sock
+
+NotifyScript    = @TARGET_PERL@ testfiles/notifyscript.pl
+SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl
+
+Server
+{
+	PidFile = testfiles/bbackupd.pid
+}
+
+BackupLocations
+{
+	Test1
+	{
+		Path = testfiles/TestDir1
+		ExcludeDir = testfiles/TestDir1/spacetest/d3
+		ExcludeFile = testfiles/TestDir1/spacetest/f2
+	}
+}
+

Copied: box/trunk/test/bbackupd/testfiles/bbackupd-snapshot.conf.in (from rev 2141, box/trunk/test/bbackupd/testfiles/bbackupd.conf.in)
===================================================================
--- box/trunk/test/bbackupd/testfiles/bbackupd-snapshot.conf.in	                        (rev 0)
+++ box/trunk/test/bbackupd/testfiles/bbackupd-snapshot.conf.in	2008-05-28 09:38:12 UTC (rev 2164)
@@ -0,0 +1,56 @@
+
+CertificateFile = testfiles/clientCerts.pem
+PrivateKeyFile = testfiles/clientPrivKey.pem
+TrustedCAsFile = testfiles/clientTrustedCAs.pem
+
+KeysFile = testfiles/bbackupd.keys
+
+DataDirectory = testfiles/bbackupd-data
+
+StoreHostname = localhost
+StorePort = 22011
+AccountNumber = 0x01234567
+
+AutomaticBackup = no
+UpdateStoreInterval = 0
+MinimumFileAge = 4
+MaxUploadWait = 24
+DeleteRedundantLocationsAfter = 10
+
+FileTrackingSizeThreshold = 1024
+DiffingUploadSizeThreshold = 1024
+
+MaximumDiffingTime = 3
+KeepAliveTime = 1
+
+ExtendedLogging = no
+ExtendedLogFile = testfiles/bbackupd.log
+
+CommandSocket = testfiles/bbackupd.sock
+
+NotifyScript    = @TARGET_PERL@ testfiles/notifyscript.pl
+SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl
+
+Server
+{
+	PidFile = testfiles/bbackupd.pid
+}
+
+BackupLocations
+{
+	Test1
+	{
+		Path = testfiles/TestDir1
+
+		ExcludeFile = testfiles/TestDir1/excluded_1
+		ExcludeFile = testfiles/TestDir1/excluded_2
+		ExcludeFilesRegex = \.excludethis$
+		ExcludeFilesRegex = EXCLUDE
+		AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis
+		ExcludeDir = testfiles/TestDir1/exclude_dir
+		ExcludeDir = testfiles/TestDir1/exclude_dir_2
+		ExcludeDirsRegex = not_this_dir
+		AlwaysIncludeDirsRegex = ALWAYSINCLUDE
+	}
+}
+

Copied: box/trunk/test/bbackupd/testfiles/bbackupd-symlink.conf.in (from rev 2141, box/trunk/test/bbackupd/testfiles/bbackupd.conf.in)
===================================================================
--- box/trunk/test/bbackupd/testfiles/bbackupd-symlink.conf.in	                        (rev 0)
+++ box/trunk/test/bbackupd/testfiles/bbackupd-symlink.conf.in	2008-05-28 09:38:12 UTC (rev 2164)
@@ -0,0 +1,55 @@
+
+CertificateFile = testfiles/clientCerts.pem
+PrivateKeyFile = testfiles/clientPrivKey.pem
+TrustedCAsFile = testfiles/clientTrustedCAs.pem
+
+KeysFile = testfiles/bbackupd.keys
+
+DataDirectory = testfiles/bbackupd-data
+
+StoreHostname = localhost
+StorePort = 22011
+AccountNumber = 0x01234567
+
+UpdateStoreInterval = 3
+MinimumFileAge = 4
+MaxUploadWait = 24
+DeleteRedundantLocationsAfter = 10
+
+FileTrackingSizeThreshold = 1024
+DiffingUploadSizeThreshold = 1024
+
+MaximumDiffingTime = 3
+KeepAliveTime = 1
+
+ExtendedLogging = no
+ExtendedLogFile = testfiles/bbackupd.log
+
+CommandSocket = testfiles/bbackupd.sock
+
+NotifyScript    = @TARGET_PERL@ testfiles/notifyscript.pl
+SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl
+
+Server
+{
+	PidFile = testfiles/bbackupd.pid
+}
+
+BackupLocations
+{
+	Test1
+	{
+		Path = testfiles/symlink-to-TestDir1
+
+		ExcludeFile = testfiles/TestDir1/excluded_1
+		ExcludeFile = testfiles/TestDir1/excluded_2
+		ExcludeFilesRegex = \.excludethis$
+		ExcludeFilesRegex = EXCLUDE
+		AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis
+		ExcludeDir = testfiles/TestDir1/exclude_dir
+		ExcludeDir = testfiles/TestDir1/exclude_dir_2
+		ExcludeDirsRegex = not_this_dir
+		AlwaysIncludeDirsRegex = ALWAYSINCLUDE
+	}
+}
+



From boxbackup-dev at fluffy.co.uk  Wed May 28 10:41:32 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 10:41:32 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2165 - box/trunk/test/bbackupd/testfiles
Message-ID: <20080528094132.75F56325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 10:41:32 +0100 (Wed, 28 May 2008)
New Revision: 2165

Modified:
   box/trunk/test/bbackupd/testfiles/notifyscript.pl.in
Log:
Add support to notifyscript to use a tag, useful for tests with 
frequently-generated messages such as backup-start.


Modified: box/trunk/test/bbackupd/testfiles/notifyscript.pl.in
===================================================================
--- box/trunk/test/bbackupd/testfiles/notifyscript.pl.in	2008-05-28 09:38:12 UTC (rev 2164)
+++ box/trunk/test/bbackupd/testfiles/notifyscript.pl.in	2008-05-28 09:41:32 UTC (rev 2165)
@@ -1,7 +1,16 @@
 #!@TARGET_PERL@
 
+my $f = 'testfiles/notifyran.'.$ARGV[0].'.';
 
-my $f = 'testfiles/notifyran.'.$ARGV[0].'.';
+if (-e 'testfiles/notifyscript.tag')
+{
+	open FILE, '< testfiles/notifyscript.tag' or die $!;
+	my $tag = <FILE>;
+	chomp $tag;
+	$f .= "$tag.";
+	close FILE;
+}
+
 my $n = 1;
 
 while(-e $f.$n)



From boxbackup-dev at fluffy.co.uk  Wed May 28 10:42:37 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 10:42:37 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2166 - box/trunk/infrastructure
Message-ID: <20080528094237.ECA32325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 10:42:37 +0100 (Wed, 28 May 2008)
New Revision: 2166

Modified:
   box/trunk/infrastructure/buildenv-testmain-template.cpp
Log:
Set program name in tests to module name (e.g. test/bbackupd).

Add option to show PID in logs (-P) in tests.


Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp
===================================================================
--- box/trunk/infrastructure/buildenv-testmain-template.cpp	2008-05-28 09:41:32 UTC (rev 2165)
+++ box/trunk/infrastructure/buildenv-testmain-template.cpp	2008-05-28 09:42:37 UTC (rev 2166)
@@ -164,6 +164,8 @@
 	// Start memory leak testing
 	MEMLEAKFINDER_START
 
+	Logging::SetProgramName(BOX_MODULE);
+
 #ifdef HAVE_GETOPT_H
 	#ifdef NDEBUG
 	int logLevel = Log::NOTICE; // need an int to do math with
@@ -181,7 +183,7 @@
 	
 	int ch;
 	
-	while ((ch = getopt_long(argc, argv, "c:d:qs:t:vTUV", longopts, NULL))
+	while ((ch = getopt_long(argc, argv, "c:d:qs:t:vPTUV", longopts, NULL))
 		!= -1)
 	{
 		switch(ch)
@@ -213,6 +215,14 @@
 			}
 			break;
 
+			#ifndef WIN32
+			case 'P':
+			{
+				Console::SetShowPID(true);
+			}
+			break;
+			#endif
+
 			case 'q':
 			{
 				if(logLevel == Log::NOTHING)



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:02:48 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:02:48 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2167 - box/trunk
Message-ID: <20080528120248.3D35E325027@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:02:47 +0100 (Wed, 28 May 2008)
New Revision: 2167

Modified:
   box/trunk/configure.ac
Log:
Check for cxxabi.h (C++ name demangling for stack traces on libstdc++)


Modified: box/trunk/configure.ac
===================================================================
--- box/trunk/configure.ac	2008-05-28 09:42:37 UTC (rev 2166)
+++ box/trunk/configure.ac	2008-05-28 12:02:47 UTC (rev 2167)
@@ -109,7 +109,7 @@
 AC_HEADER_STDC
 AC_HEADER_SYS_WAIT
 AC_CHECK_HEADERS([dlfcn.h execinfo.h getopt.h process.h pwd.h signal.h])
-AC_CHECK_HEADERS([syslog.h time.h])
+AC_CHECK_HEADERS([syslog.h time.h cxxabi.h])
 AC_CHECK_HEADERS([netinet/in.h])
 AC_CHECK_HEADERS([sys/param.h sys/socket.h sys/time.h sys/types.h sys/wait.h])
 AC_CHECK_HEADERS([sys/uio.h sys/xattr.h])



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:04:00 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:04:00 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2168 - box/trunk/infrastructure
Message-ID: <20080528120400.0D636325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:03:59 +0100 (Wed, 28 May 2008)
New Revision: 2168

Modified:
   box/trunk/infrastructure/BoxPlatform.pm.in
   box/trunk/infrastructure/makebuildenv.pl.in
Log:
Remove redundant copies of flags from make command line.

Place CXXFLAGS and LDFLAGS in Makefiles so that they can be overridden 
by users who know what they are doing.

Add LDADD_RDYNAMIC to LDFLAGS to enable -rdynamic for symbolic stack 
traces.


Modified: box/trunk/infrastructure/BoxPlatform.pm.in
===================================================================
--- box/trunk/infrastructure/BoxPlatform.pm.in	2008-05-28 12:02:47 UTC (rev 2167)
+++ box/trunk/infrastructure/BoxPlatform.pm.in	2008-05-28 12:03:59 UTC (rev 2168)
@@ -32,9 +32,8 @@
 		$build_os ne "SunOS" && $build_os ne 'GNU/kFreeBSD');
 
 	# blank extra flags by default
-	$platform_compile_line_extra = '@CPPFLAGS@ @CXXFLAGS@ @CXXFLAGS_STRICT@';
-	$platform_compile_line_extra =~ s/ -O2//;
-	$platform_link_line_extra = '@LDFLAGS@';
+	$platform_compile_line_extra = '';
+	$platform_link_line_extra = '';
 	$platform_lib_files = '@LIBS@';
 	$platform_exe_ext = '@EXEEXT@';
 

Modified: box/trunk/infrastructure/makebuildenv.pl.in
===================================================================
--- box/trunk/infrastructure/makebuildenv.pl.in	2008-05-28 12:02:47 UTC (rev 2167)
+++ box/trunk/infrastructure/makebuildenv.pl.in	2008-05-28 12:03:59 UTC (rev 2168)
@@ -515,6 +515,9 @@
 	open MAKE,">$mod/Makefile".$mk_name_extra or die "Can't open Makefile for $mod\n";
 	my $debug_link_extra = ($target_is_library)?'':'../../debug/lib/debug/debug.a';
 
+	my $default_cxxflags = '@CXXFLAGS@';
+	my $default_cxxflags =~ s/ -O2//g;
+
 	my $release_flags = "-O2";
 	if ($target_windows)
 	{
@@ -532,14 +535,20 @@
 RANLIB = @RANLIB@
 PERL = @PERL@
 WINDRES = windres
+
+CXXFLAGS = @CPPFLAGS@ $default_cxxflags @CXXFLAGS_STRICT@ \\
+	$include_paths $extra_platform_defines \\
+	-DBOX_VERSION="\\"$product_version\\""
+LDFLAGS = @LDFLAGS@ @LDADD_RDYNAMIC@
+
 .ifdef RELEASE
-CXXFLAGS = -DNDEBUG $release_flags @CXXFLAGS_STRICT@ $include_paths $extra_platform_defines -DBOX_VERSION="\\"$product_version\\""
+CXXFLAGS += -DNDEBUG $release_flags
 OUTBASE = ../../release
 OUTDIR = ../../release/$mod
 DEPENDMAKEFLAGS = -D RELEASE
 VARIENT = RELEASE
 .else
-CXXFLAGS = -g @CXXFLAGS_STRICT@ $include_paths $extra_platform_defines -DBOX_VERSION="\\"$product_version\\""
+CXXFLAGS += -g
 OUTBASE = ../../debug
 OUTDIR = ../../debug/$mod
 DEPENDMAKEFLAGS =
@@ -797,7 +806,9 @@
 		}
 	
 		# link line...
-		print MAKE "\t\$(_LINK) $link_line_extra -o $end_target $o_file_list $lib_files$lo $platform_lib_files\n";
+		print MAKE "\t\$(_LINK) \$(LDFLAGS) $link_line_extra " .
+			"-o $end_target $o_file_list " .
+			"$lib_files$lo $platform_lib_files\n";
 	}
 	# tests need to copy the test file over
 	if($type eq 'test')



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:13:52 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:13:52 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2169 - box/trunk/lib/common
Message-ID: <20080528121352.EF344325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:13:52 +0100 (Wed, 28 May 2008)
New Revision: 2169

Modified:
   box/trunk/lib/common/Box.h
Log:
Add backtrace on exception in release builds.


Modified: box/trunk/lib/common/Box.h
===================================================================
--- box/trunk/lib/common/Box.h	2008-05-28 12:03:59 UTC (rev 2168)
+++ box/trunk/lib/common/Box.h	2008-05-28 12:13:52 UTC (rev 2169)
@@ -17,13 +17,14 @@
 
 #include "BoxPlatform.h"
 
-// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL)
+// uncomment this line to enable full memory leak finding on all
+// malloc-ed blocks (at least, ones used by the STL)
 //#define MEMLEAKFINDER_FULL_MALLOC_MONITORING
 
-#ifndef NDEBUG
-	#ifdef HAVE_EXECINFO_H
-		#define SHOW_BACKTRACE_ON_EXCEPTION
-	#endif
+// Show backtraces on exceptions in release builds until further notice
+// (they are only logged at TRACE level anyway)
+#ifdef HAVE_EXECINFO_H
+	#define SHOW_BACKTRACE_ON_EXCEPTION
 #endif
 
 #ifdef SHOW_BACKTRACE_ON_EXCEPTION
@@ -43,7 +44,8 @@
 	void BoxDebugAssertFailed(const char *cond, const char *file, int line);
 	#define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}}
 
-	// Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not
+	// Note that syslog tracing is independent of BoxDebugTraceOn,
+	// but stdout tracing is not
 	extern bool BoxDebugTraceToSyslog;
 	#define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;}
 	extern bool BoxDebugTraceToStdout;



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:14:38 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:14:38 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2170 - box/trunk/lib/backupclient
Message-ID: <20080528121438.D583B325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:14:38 +0100 (Wed, 28 May 2008)
New Revision: 2170

Modified:
   box/trunk/lib/backupclient/BackupClientRestore.cpp
Log:
Initialise the exists variable and fix its type.


Modified: box/trunk/lib/backupclient/BackupClientRestore.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupClientRestore.cpp	2008-05-28 12:13:52 UTC (rev 2169)
+++ box/trunk/lib/backupclient/BackupClientRestore.cpp	2008-05-28 12:14:38 UTC (rev 2170)
@@ -595,7 +595,7 @@
 				
 				// Save restore info?
 				int64_t fileSize;
-				int exists;
+				bool exists = false;
 
 				try
 				{



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:15:56 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:15:56 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2171 - box/trunk/lib/server
Message-ID: <20080528121556.AD1FC325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:15:56 +0100 (Wed, 28 May 2008)
New Revision: 2171

Added:
   box/trunk/lib/server/ServerControl.cpp
Modified:
   box/trunk/lib/server/ServerControl.h
Log:
Move ServerControl functions out of line and into their own source file.


Copied: box/trunk/lib/server/ServerControl.cpp (from rev 2162, box/trunk/lib/server/ServerControl.h)
===================================================================
--- box/trunk/lib/server/ServerControl.cpp	                        (rev 0)
+++ box/trunk/lib/server/ServerControl.cpp	2008-05-28 12:15:56 UTC (rev 2171)
@@ -0,0 +1,198 @@
+#include "Box.h"
+
+#include <errno.h>
+
+#ifdef HAVE_SYS_TYPES_H
+	#include <sys/types.h>
+#endif
+
+#ifdef HAVE_SIGNAL_H
+	#include <signal.h>
+#endif
+
+#include "ServerControl.h"
+#include "Test.h"
+
+#ifdef WIN32
+
+#include "WinNamedPipeStream.h"
+#include "IOStreamGetLine.h"
+#include "BoxPortsAndFiles.h"
+
+static std::string sPipeName;
+
+void SetNamedPipeName(const std::string& rPipeName)
+{
+	sPipeName = rPipeName;
+}
+
+bool SendCommands(const std::string& rCmd)
+{
+	WinNamedPipeStream connection;
+
+	try
+	{
+		connection.Connect(sPipeName);
+	}
+	catch(...)
+	{
+		BOX_ERROR("Failed to connect to daemon control socket");
+		return false;
+	}
+
+	// For receiving data
+	IOStreamGetLine getLine(connection);
+	
+	// Wait for the configuration summary
+	std::string configSummary;
+	if(!getLine.GetLine(configSummary))
+	{
+		BOX_ERROR("Failed to receive configuration summary from daemon");
+		return false;
+	}
+
+	// Was the connection rejected by the server?
+	if(getLine.IsEOF())
+	{
+		BOX_ERROR("Server rejected the connection");
+		return false;
+	}
+
+	// Decode it
+	int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait;
+	if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", 
+			&autoBackup, &updateStoreInterval, 
+			&minimumFileAge, &maxUploadWait) != 4)
+	{
+		BOX_ERROR("Config summary didn't decode");
+		return false;
+	}
+
+	std::string cmds;
+	bool expectResponse;
+
+	if (rCmd != "")
+	{
+		cmds = rCmd;
+		cmds += "\nquit\n";
+		expectResponse = true;
+	}
+	else
+	{
+		cmds = "quit\n";
+		expectResponse = false;
+	}
+	
+	connection.Write(cmds.c_str(), cmds.size());
+	
+	// Read the response
+	std::string line;
+	bool statusOk = !expectResponse;
+
+	while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line))
+	{
+		// Is this an OK or error line?
+		if (line == "ok")
+		{
+			statusOk = true;
+		}
+		else if (line == "error")
+		{
+			BOX_ERROR(rCmd);
+			break;
+		}
+		else
+		{
+			BOX_WARNING("Unexpected response to command '" <<
+				rCmd << "': " << line)
+		}
+	}
+	
+	return statusOk;
+}
+
+bool HUPServer(int pid)
+{
+	return SendCommands("reload");
+}
+
+bool KillServerInternal(int pid)
+{
+	HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid);
+	if (hProcess == NULL)
+	{
+		BOX_ERROR("Failed to open process " << pid << ": " <<
+			GetErrorMessage(GetLastError()));
+		return false;
+	}
+
+	if (!TerminateProcess(hProcess, 1))
+	{
+		BOX_ERROR("Failed to terminate process " << pid << ": " <<
+			GetErrorMessage(GetLastError()));
+		CloseHandle(hProcess);
+		return false;
+	}
+
+	CloseHandle(hProcess);
+	return true;
+}
+
+#else // !WIN32
+
+bool HUPServer(int pid)
+{
+	if(pid == 0) return false;
+	return ::kill(pid, SIGHUP) == 0;
+}
+
+bool KillServerInternal(int pid)
+{
+	if(pid == 0 || pid == -1) return false;
+	bool killed = (::kill(pid, SIGTERM) == 0);
+	if (!killed)
+	{
+		BOX_LOG_SYS_ERROR("Failed to kill process " << pid);
+	}
+	TEST_THAT(killed);
+	return killed;
+}
+
+#endif // WIN32
+
+bool KillServer(int pid)
+{
+	if (!KillServerInternal(pid))
+	{
+		return false;
+	}
+
+	for (int i = 0; i < 30; i++)
+	{
+		if (i == 0) 
+		{
+			printf("Waiting for server to die: ");
+		}
+
+		printf(".");
+		fflush(stdout);
+
+		if (!ServerIsAlive(pid)) break;
+		::sleep(1);
+		if (!ServerIsAlive(pid)) break;
+	}
+
+	if (!ServerIsAlive(pid))
+	{
+		printf(" done.\n");
+	}
+	else
+	{
+		printf(" failed!\n");
+	}
+
+	fflush(stdout);
+
+	return !ServerIsAlive(pid);
+}
+

Modified: box/trunk/lib/server/ServerControl.h
===================================================================
--- box/trunk/lib/server/ServerControl.h	2008-05-28 12:14:38 UTC (rev 2170)
+++ box/trunk/lib/server/ServerControl.h	2008-05-28 12:15:56 UTC (rev 2171)
@@ -3,187 +3,16 @@
 
 #include "Test.h"
 
+bool HUPServer(int pid);
+bool KillServer(int pid);
+
 #ifdef WIN32
+	#include "WinNamedPipeStream.h"
+	#include "IOStreamGetLine.h"
+	#include "BoxPortsAndFiles.h"
 
-#include "WinNamedPipeStream.h"
-#include "IOStreamGetLine.h"
-#include "BoxPortsAndFiles.h"
-
-static std::string sPipeName;
-
-static void SetNamedPipeName(const std::string& rPipeName)
-{
-	sPipeName = rPipeName;
-}
-
-static bool SendCommands(const std::string& rCmd)
-{
-	WinNamedPipeStream connection;
-
-	try
-	{
-		connection.Connect(sPipeName);
-	}
-	catch(...)
-	{
-		BOX_ERROR("Failed to connect to daemon control socket");
-		return false;
-	}
-
-	// For receiving data
-	IOStreamGetLine getLine(connection);
-	
-	// Wait for the configuration summary
-	std::string configSummary;
-	if(!getLine.GetLine(configSummary))
-	{
-		BOX_ERROR("Failed to receive configuration summary from daemon");
-		return false;
-	}
-
-	// Was the connection rejected by the server?
-	if(getLine.IsEOF())
-	{
-		BOX_ERROR("Server rejected the connection");
-		return false;
-	}
-
-	// Decode it
-	int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait;
-	if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", 
-			&autoBackup, &updateStoreInterval, 
-			&minimumFileAge, &maxUploadWait) != 4)
-	{
-		BOX_ERROR("Config summary didn't decode");
-		return false;
-	}
-
-	std::string cmds;
-	bool expectResponse;
-
-	if (rCmd != "")
-	{
-		cmds = rCmd;
-		cmds += "\nquit\n";
-		expectResponse = true;
-	}
-	else
-	{
-		cmds = "quit\n";
-		expectResponse = false;
-	}
-	
-	connection.Write(cmds.c_str(), cmds.size());
-	
-	// Read the response
-	std::string line;
-	bool statusOk = !expectResponse;
-
-	while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line))
-	{
-		// Is this an OK or error line?
-		if (line == "ok")
-		{
-			statusOk = true;
-		}
-		else if (line == "error")
-		{
-			BOX_ERROR(rCmd);
-			break;
-		}
-		else
-		{
-			BOX_WARNING("Unexpected response to command '" <<
-				rCmd << "': " << line)
-		}
-	}
-	
-	return statusOk;
-}
-
-inline bool HUPServer(int pid)
-{
-	return SendCommands("reload");
-}
-
-inline bool KillServerInternal(int pid)
-{
-	HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid);
-	if (hProcess == NULL)
-	{
-		BOX_ERROR("Failed to open process " << pid << ": " <<
-			GetErrorMessage(GetLastError()));
-		return false;
-	}
-
-	if (!TerminateProcess(hProcess, 1))
-	{
-		BOX_ERROR("Failed to terminate process " << pid << ": " <<
-			GetErrorMessage(GetLastError()));
-		CloseHandle(hProcess);
-		return false;
-	}
-
-	CloseHandle(hProcess);
-	return true;
-}
-
-#else // !WIN32
-
-inline bool HUPServer(int pid)
-{
-	if(pid == 0) return false;
-	return ::kill(pid, SIGHUP) == 0;
-}
-
-inline bool KillServerInternal(int pid)
-{
-	if(pid == 0 || pid == -1) return false;
-	bool killed = (::kill(pid, SIGTERM) == 0);
-	if (!killed)
-	{
-		BOX_LOG_SYS_ERROR("Failed to kill process " << pid);
-	}
-	TEST_THAT(killed);
-	return killed;
-}
-
+	void SetNamedPipeName(const std::string& rPipeName);
+	// bool SendCommands(const std::string& rCmd);
 #endif // WIN32
 
-inline bool KillServer(int pid)
-{
-	if (!KillServerInternal(pid))
-	{
-		return false;
-	}
-
-	for (int i = 0; i < 30; i++)
-	{
-		if (i == 0) 
-		{
-			printf("Waiting for server to die: ");
-		}
-
-		printf(".");
-		fflush(stdout);
-
-		if (!ServerIsAlive(pid)) break;
-		::sleep(1);
-		if (!ServerIsAlive(pid)) break;
-	}
-
-	if (!ServerIsAlive(pid))
-	{
-		printf(" done.\n");
-	}
-	else
-	{
-		printf(" failed!\n");
-	}
-
-	fflush(stdout);
-
-	return !ServerIsAlive(pid);
-}
-
 #endif // SERVER_CONTROL_H



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:33:42 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:33:42 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2172 - box/trunk/lib/server
Message-ID: <20080528123342.9A60A325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:33:42 +0100 (Wed, 28 May 2008)
New Revision: 2172

Modified:
   box/trunk/lib/server/makeprotocol.pl.in
Log:
Add a GetMessage() method to Protocol objects to return a stringified 
message as a string.

Fix a typo in a comment.


Modified: box/trunk/lib/server/makeprotocol.pl.in
===================================================================
--- box/trunk/lib/server/makeprotocol.pl.in	2008-05-28 12:15:56 UTC (rev 2171)
+++ box/trunk/lib/server/makeprotocol.pl.in	2008-05-28 12:33:42 UTC (rev 2172)
@@ -178,6 +178,9 @@
 // Auto-generated file -- do not edit
 
 #include "Box.h"
+
+#include <sstream>
+
 #include "$h_filename"
 #include "IOStream.h"
 
@@ -273,7 +276,7 @@
 
 if($derive_objects_from ne 'ProtocolObject')
 {	
-	# output a definition for the protocol object derviced class
+	# output a definition for the protocol object derived class
 	print H <<__E;
 class ${protocol_name}ProtocolServer;
 	
@@ -338,6 +341,7 @@
 	if(obj_is_type($cmd,'IsError'))
 	{
 		print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n";
+		print H "\tstd::string GetMessage() const;\n";
 	}
 	if($type eq 'Server' && obj_is_type($cmd, 'Command'))
 	{
@@ -498,8 +502,29 @@
 	rSubTypeOut = m$mem_subtype;
 	return true;
 }
+std::string ${class}GetMessage() const
+{
+	switch(m$mem_subtype)
+	{
 __E
+		foreach my $const (@{$cmd_constants{$cmd}})
+		{
+			next unless $const =~ /^Err_(.*)/;
+			my $shortname = $1;
+			$const =~ s/ = .*//;
+			print CPP <<__E;
+		case $const: return "$shortname";
+__E
+		}
+		print CPP <<__E;
+		default:
+			std::ostringstream out;
+			out << "Unknown subtype " << m$mem_subtype;
+			return out.str();
 	}
+}
+__E
+	}
 
 	if($implement_syslog)
 	{
@@ -890,13 +915,15 @@
 		if(preply->IsError(type, subType))
 		{
 			SetError(type, subType);
-			BOX_WARNING("Protocol: Received error " << type <<
-				"/" << subType);
+			BOX_WARNING("$cmd command failed: received error " <<
+				((${classname_base}Error&)*preply).GetMessage());
 		}
 		else
 		{
 			SetError(Protocol::UnknownError, Protocol::UnknownError);
-			BOX_WARNING("Protocol: Received unknown error");
+			BOX_WARNING("$cmd command failed: received "
+				"unexpected response type " <<
+				preply->GetType());
 		}
 		
 		// Throw an exception



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:36:05 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:36:05 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2173 - box/trunk/lib/common
Message-ID: <20080528123605.801D7325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:36:05 +0100 (Wed, 28 May 2008)
New Revision: 2173

Modified:
   box/trunk/lib/common/DebugMemLeakFinder.cpp
Log:
Reformat for readability.


Modified: box/trunk/lib/common/DebugMemLeakFinder.cpp
===================================================================
--- box/trunk/lib/common/DebugMemLeakFinder.cpp	2008-05-28 12:33:42 UTC (rev 2172)
+++ box/trunk/lib/common/DebugMemLeakFinder.cpp	2008-05-28 12:36:05 UTC (rev 2173)
@@ -344,13 +344,27 @@
 
 	ASSERT(!sTrackingDataDestroyed);
 
-	for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i)
+	for(std::map<void *, MallocBlockInfo>::const_iterator
+		i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i)
 	{
-		if(is_leak(i->first)) ::fprintf(file, "Block 0x%p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line);
+		if(is_leak(i->first))
+		{
+			::fprintf(file, "Block %p size %d allocated at "
+				"%s:%d\n", i->first, i->second.size,
+				i->second.file, i->second.line);
+		}
 	}
-	for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i)
+
+	for(std::map<void *, ObjectInfo>::const_iterator
+		i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i)
 	{
-		if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line);
+		if(is_leak(i->first))
+		{
+			::fprintf(file, "Object%s %p size %d allocated at "
+				"%s:%d\n", i->second.array?" []":"",
+				i->first, i->second.size, i->second.file,
+				i->second.line);
+		}
 	}
 }
 



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:36:53 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:36:53 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2174 - box/trunk/lib/common
Message-ID: <20080528123653.34C5E325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:36:53 +0100 (Wed, 28 May 2008)
New Revision: 2174

Modified:
   box/trunk/lib/common/Test.cpp
Log:
Separate checks for file existing and file not empty.


Modified: box/trunk/lib/common/Test.cpp
===================================================================
--- box/trunk/lib/common/Test.cpp	2008-05-28 12:36:05 UTC (rev 2173)
+++ box/trunk/lib/common/Test.cpp	2008-05-28 12:36:53 UTC (rev 2174)
@@ -25,6 +25,12 @@
 bool TestFileExists(const char *Filename)
 {
 	struct stat st;
+	return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0;
+}
+
+bool TestFileNotEmpty(const char *Filename)
+{
+	struct stat st;
 	return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0 &&
 		st.st_size > 0;
 }
@@ -102,7 +108,7 @@
 
 int ReadPidFile(const char *pidFile)
 {
-	if(!TestFileExists(pidFile))
+	if(!TestFileNotEmpty(pidFile))
 	{
 		TEST_FAIL_WITH_MESSAGE("Server didn't save PID file "
 			"(perhaps one was already running?)");	
@@ -194,7 +200,7 @@
 
 	for (int i = 0; i < 15; i++)
 	{
-		if (TestFileExists(pidFile))	
+		if (TestFileNotEmpty(pidFile))	
 		{
 			break;
 		}
@@ -223,7 +229,7 @@
 	}
 	#endif
 
-	if (!TestFileExists(pidFile))
+	if (!TestFileNotEmpty(pidFile))
 	{
 		::fprintf(stdout, " timed out!\n");
 		TEST_FAIL_WITH_MESSAGE("Server didn't save PID file");	



From boxbackup-dev at fluffy.co.uk  Wed May 28 13:37:49 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 13:37:49 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2175 - box/trunk/lib/common
Message-ID: <20080528123749.A07DF325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 13:37:49 +0100 (Wed, 28 May 2008)
New Revision: 2175

Modified:
   box/trunk/lib/common/Logging.cpp
   box/trunk/lib/common/Logging.h
Log:
Add static Logging::GetNamedLevel() method to return a log level 
specified by name as a string.


Modified: box/trunk/lib/common/Logging.cpp
===================================================================
--- box/trunk/lib/common/Logging.cpp	2008-05-28 12:36:53 UTC (rev 2174)
+++ box/trunk/lib/common/Logging.cpp	2008-05-28 12:37:49 UTC (rev 2175)
@@ -179,6 +179,23 @@
 	sContextSet = true;
 }
 
+Log::Level Logging::GetNamedLevel(const std::string& rName)
+{
+	if      (rName == "nothing") { return Log::NOTHING; }
+	else if (rName == "fatal")   { return Log::FATAL; }
+	else if (rName == "error")   { return Log::ERROR; }
+	else if (rName == "warning") { return Log::WARNING; }
+	else if (rName == "notice")  { return Log::NOTICE; }
+	else if (rName == "info")    { return Log::INFO; }
+	else if (rName == "trace")   { return Log::TRACE; }
+	else if (rName == "everything") { return Log::EVERYTHING; }
+	else
+	{
+		BOX_ERROR("Unknown verbosity level: " << rName);
+		return Log::INVALID;
+	}
+}
+
 void Logging::ClearContext()
 {
 	sContextSet = false;
@@ -350,6 +367,7 @@
 	switch(level)
 	{
 		case Log::NOTHING:    /* fall through */
+		case Log::INVALID:    /* fall through */
 		case Log::FATAL:      syslogLevel = LOG_CRIT;    break;
 		case Log::ERROR:      syslogLevel = LOG_ERR;     break;
 		case Log::WARNING:    syslogLevel = LOG_WARNING; break;

Modified: box/trunk/lib/common/Logging.h
===================================================================
--- box/trunk/lib/common/Logging.h	2008-05-28 12:36:53 UTC (rev 2174)
+++ box/trunk/lib/common/Logging.h	2008-05-28 12:37:49 UTC (rev 2175)
@@ -93,7 +93,8 @@
 		NOTICE,
 		INFO,
 		TRACE, 
-		EVERYTHING
+		EVERYTHING,
+		INVALID = -1,
 	};
 }
 
@@ -225,11 +226,30 @@
 	static void SetContext(std::string context);
 	static void ClearContext();
 	static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; }
+	static Log::Level GetGlobalLevel() { return sGlobalLevel; }
+	static Log::Level GetNamedLevel(const std::string& rName);
 	static bool IsEnabled(Log::Level level)
 	{
 		return (int)sGlobalLevel >= (int)level;
 	}
 	static void SetProgramName(const std::string& rProgramName);
+
+	class Guard
+	{
+		private:
+		Log::Level mOldLevel;
+
+		public:
+		Guard(Log::Level newLevel)
+		{
+			mOldLevel = Logging::GetGlobalLevel();
+			Logging::SetGlobalLevel(newLevel);
+		}
+		~Guard()
+		{
+			Logging::SetGlobalLevel(mOldLevel);
+		}
+	};
 };
 
 #endif // LOGGING__H



From boxbackup-dev at fluffy.co.uk  Wed May 28 15:27:21 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 15:27:21 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2176 - box/trunk/bin/bbstored
Message-ID: <20080528142721.69CB3325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 15:27:21 +0100 (Wed, 28 May 2008)
New Revision: 2176

Modified:
   box/trunk/bin/bbstored/HousekeepStoreAccount.cpp
   box/trunk/bin/bbstored/HousekeepStoreAccount.h
Log:
Reformat for readability.

Log removal of directories by housekeeping.


Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.cpp
===================================================================
--- box/trunk/bin/bbstored/HousekeepStoreAccount.cpp	2008-05-28 12:37:49 UTC (rev 2175)
+++ box/trunk/bin/bbstored/HousekeepStoreAccount.cpp	2008-05-28 14:27:21 UTC (rev 2176)
@@ -85,16 +85,19 @@
 {
 	// Attempt to lock the account
 	std::string writeLockFilename;
-	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFilename);
+	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet,
+		writeLockFilename);
 	NamedLock writeLock;
-	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */))
+	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
+		0600 /* restrictive file permissions */))
 	{
 		// Couldn't lock the account -- just stop now
 		return;
 	}
 
 	// Load the store info to find necessary info for the housekeeping
-	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, false /* Read/Write */));
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID,
+		mStoreRoot, mStoreDiscSet, false /* Read/Write */));
 
 	// Calculate how much should be deleted
 	mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
@@ -104,14 +107,18 @@
 	}
 
 	// Scan the directory for potential things to delete
-	// This will also remove elegiable items marked with RemoveASAP
+	// This will also remove eligible items marked with RemoveASAP
 	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
 
-	// If scan directory stopped for some reason, probably parent instructed to teminate, stop now.
+	// If scan directory stopped for some reason, probably parent
+	// instructed to terminate, stop now.
 	if(!continueHousekeeping)
 	{
-		// If any files were marked "delete now", then update the size of the store.
-		if(mBlocksUsedDelta != 0 || mBlocksInOldFilesDelta != 0 || mBlocksInDeletedFilesDelta != 0)
+		// If any files were marked "delete now", then update
+		// the size of the store.
+		if(mBlocksUsedDelta != 0 || 
+			mBlocksInOldFilesDelta != 0 ||
+			mBlocksInDeletedFilesDelta != 0)
 		{
 			info->ChangeBlocksUsed(mBlocksUsedDelta);
 			info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
@@ -124,8 +131,8 @@
 		return;
 	}
 
-	// Log any difference in opinion between the values recorded in the store info, and
-	// the values just calculated for space usage.
+	// Log any difference in opinion between the values recorded in
+	// the store info, and the values just calculated for space usage.
 	// BLOCK
 	{
 		int64_t used = info->GetBlocksUsed();
@@ -133,9 +140,12 @@
 		int64_t usedDeleted = info->GetBlocksInDeletedFiles();
 		int64_t usedDirectories = info->GetBlocksInDirectories();
 
-		// If the counts were wrong, taking into account RemoveASAP items deleted, log a message
-		if((used + mBlocksUsedDelta) != mBlocksUsed || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
-			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories)
+		// If the counts were wrong, taking into account RemoveASAP 
+		// items deleted, log a message
+		if((used + mBlocksUsedDelta) != mBlocksUsed 
+			|| (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
+			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles 
+			|| usedDirectories != mBlocksInDirectories)
 		{
 			// Log this
 			BOX_ERROR("Housekeeping on account " << 
@@ -153,18 +163,25 @@
 		}
 		
 		// If the current values don't match, store them
-		if(used != mBlocksUsed || usedOld != mBlocksInOldFiles
-			|| usedDeleted != mBlocksInDeletedFiles || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
+		if(used != mBlocksUsed 
+			|| usedOld != mBlocksInOldFiles
+			|| usedDeleted != mBlocksInDeletedFiles 
+			|| usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
 		{	
 			// Set corrected values in store info
-			info->CorrectAllUsedValues(mBlocksUsed, mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories + mBlocksInDirectoriesDelta);
+			info->CorrectAllUsedValues(mBlocksUsed,
+				mBlocksInOldFiles, mBlocksInDeletedFiles,
+				mBlocksInDirectories + mBlocksInDirectoriesDelta);
 			info->Save();
 		}
 	}
 	
-	// Reset the delta counts for files, as they will include RemoveASAP flagged files deleted
-	// during the initial scan.
-	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;	// keep for reporting
+	// Reset the delta counts for files, as they will include 
+	// RemoveASAP flagged files deleted during the initial scan.
+
+	// keep for reporting
+	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+
 	mBlocksUsedDelta = 0;
 	mBlocksInOldFilesDelta = 0;
 	mBlocksInDeletedFilesDelta = 0;
@@ -172,7 +189,8 @@
 	// Go and delete items from the accounts
 	bool deleteInterrupted = DeleteFiles();
 	
-	// If that wasn't interrupted, remove any empty directories which are also marked as deleted in their containing directory
+	// If that wasn't interrupted, remove any empty directories which 
+	// are also marked as deleted in their containing directory
 	if(!deleteInterrupted)
 	{
 		deleteInterrupted = DeleteEmptyDirectories();
@@ -190,8 +208,9 @@
 			(deleteInterrupted?" and was interrupted":""));
 	}
 	
-	// Make sure the delta's won't cause problems if the counts are really wrong, and
-	// it wasn't fixed because the store was updated during the scan.
+	// Make sure the delta's won't cause problems if the counts are 
+	// really wrong, and it wasn't fixed because the store was 
+	// updated during the scan.
 	if(mBlocksUsedDelta < (0 - info->GetBlocksUsed()))
 	{
 		mBlocksUsedDelta = (0 - info->GetBlocksUsed());
@@ -218,7 +237,8 @@
 	// Save the store info back
 	info->Save();
 	
-	// Explicity release the lock (would happen automatically on going out of scope, included for code clarity)
+	// Explicity release the lock (would happen automatically on 
+	// going out of scope, included for code clarity)
 	writeLock.ReleaseLock();
 }
 
@@ -243,8 +263,9 @@
 //
 // Function
 //		Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
-//		Purpose: Private. Scan a directory for potenitally deleteable items, and
-//				 add them to the list. Returns true if the scan should continue.
+//		Purpose: Private. Scan a directory for potentially deleteable
+//			 items, and add them to the list. Returns true if the
+//			 scan should continue.
 //		Created: 11/12/03
 //
 // --------------------------------------------------------------------------
@@ -253,9 +274,12 @@
 #ifndef WIN32
 	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
 	{
-		mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+		mCountUntilNextInterprocessMsgCheck =
+			POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+
 		// Check for having to stop
-		if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is locked
+		// Include account ID here as the specified account is locked
+		if(mrDaemon.CheckForInterProcessMsg(mAccountID))
 		{
 			// Need to abort now
 			return false;
@@ -268,7 +292,8 @@
 	MakeObjectFilename(ObjectID, objectFilename);
 
 	// Open it.
-	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename));
+	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet,
+		objectFilename));
 	
 	// Add the size of the directory on disc to the size being calculated
 	int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
@@ -290,8 +315,8 @@
 	
 	// BLOCK
 	{
-		// Remove any files which are marked for removal as soon as they become old
-		// or deleted.
+		// Remove any files which are marked for removal as soon
+		// as they become old or deleted.
 		bool deletedSomething = false;
 		do
 		{
@@ -364,6 +389,9 @@
 				d.mSizeInBlocks = en->GetSizeInBlocks();
 				d.mMarkNumber = en->GetMarkNumber();
 				d.mVersionAgeWithinMark = enVersionAge;
+				d.mIsFlagDeleted = (enFlags &
+					BackupStoreDirectory::Entry::Flags_Deleted)
+					? true : false;
 				
 				// Add it to the list
 				mPotentialDeletions.insert(d);
@@ -541,6 +569,10 @@
 		
 		// Delete the file
 		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+		BOX_INFO("Housekeeping removed " <<
+			(i->mIsFlagDeleted ? "deleted" : "old") <<
+			" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
+			" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
 		
 		// Stop if the deletion target has been matched or exceeded
 		// (checking here rather than at the beginning will tend to reduce the
@@ -786,99 +818,115 @@
 				continue;
 			}
 
-			// Load up the directory to potentially delete
-			std::string dirFilename;
-			BackupStoreDirectory dir;
-			int64_t dirSizeInBlocks = 0;
-			{
-				MakeObjectFilename(*i, dirFilename);
-				// Check it actually exists (just in case it gets added twice to the list)
-				if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
-				{
-					// doesn't exist, next!
-					continue;
-				}
-				// load
-				std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
-				dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
-				dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
-			}
+			DeleteEmptyDirectory(*i, toExamine);
+		}
 
-			// Make sure this directory is actually empty
-			if(dir.GetNumberOfEntries() != 0)
-			{
-				// Not actually empty, try next one
-				continue;
-			}
+		// Remove contents of empty directories
+		mEmptyDirectories.clear();
+		// Swap in new, so it's examined next time round
+		mEmptyDirectories.swap(toExamine);
+	}
+	
+	// Not interrupted
+	return false;
+}
 
-			// Candiate for deletion... open containing directory
-			std::string containingDirFilename;
-			BackupStoreDirectory containingDir;
-			int64_t containingDirSizeInBlocksOrig = 0;
-			{
-				MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
-				std::auto_ptr<RaidFileRead> containingDirStream(RaidFileRead::Open(mStoreDiscSet, containingDirFilename));
-				containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks();
-				containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite);
-			}
+void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
+	std::vector<int64_t>& rToExamine)
+{
+	// Load up the directory to potentially delete
+	std::string dirFilename;
+	BackupStoreDirectory dir;
+	int64_t dirSizeInBlocks = 0;
 
-			// Find the entry
-			BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID());
-			if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
-			{
-				// Should be deleted
-				containingDir.DeleteEntry(dir.GetObjectID());
+	// BLOCK
+	{
+		MakeObjectFilename(dirId, dirFilename);
+		// Check it actually exists (just in case it gets
+		// added twice to the list)
+		if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
+		{
+			// doesn't exist, next!
+			return;
+		}
+		// load
+		std::auto_ptr<RaidFileRead> dirStream(
+			RaidFileRead::Open(mStoreDiscSet, dirFilename));
+		dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+		dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+	}
 
-				// Is the containing dir now a candidate for deletion?
-				if(containingDir.GetNumberOfEntries() == 0)
-				{
-					toExamine.push_back(containingDir.GetObjectID());
-				}
+	// Make sure this directory is actually empty
+	if(dir.GetNumberOfEntries() != 0)
+	{
+		// Not actually empty, try next one
+		return;
+	}
 
-				// Write revised parent directory
-				RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename);
-				writeDir.Open(true /* allow overwriting */);
-				containingDir.WriteToStream(writeDir);
+	// Candidate for deletion... open containing directory
+	std::string containingDirFilename;
+	BackupStoreDirectory containingDir;
+	int64_t containingDirSizeInBlocksOrig = 0;
+	{
+		MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
+		std::auto_ptr<RaidFileRead> containingDirStream(
+			RaidFileRead::Open(mStoreDiscSet,
+				containingDirFilename));
+		containingDirSizeInBlocksOrig = 
+			containingDirStream->GetDiscUsageInBlocks();
+		containingDir.ReadFromStream(*containingDirStream,
+			IOStream::TimeOutInfinite);
+	}
 
-				// get the disc usage (must do this before commiting it)
-				int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+	// Find the entry
+	BackupStoreDirectory::Entry *pdirentry = 
+		containingDir.FindEntryByID(dir.GetObjectID());
+	if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
+	{
+		// Should be deleted
+		containingDir.DeleteEntry(dir.GetObjectID());
 
-				// Commit directory
-				writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+		// Is the containing dir now a candidate for deletion?
+		if(containingDir.GetNumberOfEntries() == 0)
+		{
+			rToExamine.push_back(containingDir.GetObjectID());
+		}
 
-				// adjust usage counts for this directory
-				if(dirSize > 0)
-				{
-					int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
-					mBlocksUsedDelta += adjust;
-					mBlocksInDirectoriesDelta += adjust;
-				}
+		// Write revised parent directory
+		RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename);
+		writeDir.Open(true /* allow overwriting */);
+		containingDir.WriteToStream(writeDir);
 
-				// Delete the directory itself
-				{
-					RaidFileWrite del(mStoreDiscSet, dirFilename);
-					del.Delete();
-				}
+		// get the disc usage (must do this before commiting it)
+		int64_t dirSize = writeDir.GetDiscUsageInBlocks();
 
-				// And adjust usage counts for the directory that's just been deleted
-				mBlocksUsedDelta -= dirSizeInBlocks;
-				mBlocksInDirectoriesDelta -= dirSizeInBlocks;
+		// Commit directory
+		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
 
-				// Update count
-				++mEmptyDirectoriesDeleted;
-			}
+		// adjust usage counts for this directory
+		if(dirSize > 0)
+		{
+			int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
+			mBlocksUsedDelta += adjust;
+			mBlocksInDirectoriesDelta += adjust;
 		}
 
-		// Remove contents of empty directories
-		mEmptyDirectories.clear();
-		// Swap in new, so it's examined next time round
-		mEmptyDirectories.swap(toExamine);
-	}
-	
-	// Not interrupted
-	return false;
-}
+		// Delete the directory itself
+		{
+			RaidFileWrite del(mStoreDiscSet, dirFilename);
+			del.Delete();
+		}
 
+		BOX_INFO("Housekeeping removed empty deleted dir " <<
+			BOX_FORMAT_OBJECTID(dirId));
 
+		// And adjust usage counts for the directory that's
+		// just been deleted
+		mBlocksUsedDelta -= dirSizeInBlocks;
+		mBlocksInDirectoriesDelta -= dirSizeInBlocks;
 
+		// Update count
+		++mEmptyDirectoriesDeleted;
+	}
+}
 

Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.h
===================================================================
--- box/trunk/bin/bbstored/HousekeepStoreAccount.h	2008-05-28 12:37:49 UTC (rev 2175)
+++ box/trunk/bin/bbstored/HousekeepStoreAccount.h	2008-05-28 14:27:21 UTC (rev 2176)
@@ -42,6 +42,8 @@
 	bool ScanDirectory(int64_t ObjectID);
 	bool DeleteFiles();
 	bool DeleteEmptyDirectories();
+	void DeleteEmptyDirectory(int64_t dirId,
+		std::vector<int64_t>& rToExamine);
 	void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
 
 private:
@@ -52,6 +54,7 @@
 		int64_t mSizeInBlocks;
 		int32_t mMarkNumber;
 		int32_t mVersionAgeWithinMark;	// 0 == current, 1 latest old version, etc
+		bool    mIsFlagDeleted; // false for files flagged "Old"
 	} DelEn;
 	
 	struct DelEnCompare



From boxbackup-dev at fluffy.co.uk  Wed May 28 15:28:16 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 15:28:16 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2177 - box/trunk/lib/server
Message-ID: <20080528142816.3E935325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 15:28:16 +0100 (Wed, 28 May 2008)
New Revision: 2177

Modified:
   box/trunk/lib/server/Daemon.cpp
   box/trunk/lib/server/Daemon.h
Log:
Move loading configuration into a separate method.

Add -W<level> option to set warning level explicitly.


Modified: box/trunk/lib/server/Daemon.cpp
===================================================================
--- box/trunk/lib/server/Daemon.cpp	2008-05-28 14:27:21 UTC (rev 2176)
+++ box/trunk/lib/server/Daemon.cpp	2008-05-28 14:28:16 UTC (rev 2177)
@@ -112,7 +112,7 @@
 	#else // !WIN32
 		"DFkP"
 	#endif
-		"hqvVt:TU";
+		"hqt:TUvVW:";
 }
 
 void Daemon::Usage()
@@ -137,6 +137,7 @@
 	"  -q         Run more quietly, reduce verbosity level by one, can repeat\n"
 	"  -v         Run more verbosely, increase verbosity level by one, can repeat\n"
 	"  -V         Run at maximum verbosity\n"
+	"  -W <level> Set verbosity to error/warning/notice/info/trace/everything\n"
 	"  -t <tag>   Tag console output with specified marker\n"
 	"  -T         Timestamp console output\n"
 	"  -U         Timestamp console output with microseconds\n";
@@ -237,6 +238,17 @@
 		}
 		break;
 
+		case 'W':
+		{
+			mLogLevel = Logging::GetNamedLevel(optarg);
+			if (mLogLevel == Log::INVALID)
+			{
+				BOX_FATAL("Invalid logging level");
+				return 2;
+			}
+		}
+		break;
+
 		case 't':
 		{
 			Console::SetTag(optarg);
@@ -353,6 +365,62 @@
 // --------------------------------------------------------------------------
 //
 // Function
+//		Name:    Daemon::Configure(const std::string& rConfigFileName)
+//		Purpose: Loads daemon configuration. Useful when you have
+//			 a local Daemon object and don't intend to fork()
+//			 or call Main().
+//		Created: 2008/04/19
+//
+// --------------------------------------------------------------------------
+
+bool Daemon::Configure(const std::string& rConfigFileName)
+{
+	mConfigFileName = rConfigFileName;
+
+	// Load the configuration file.
+	std::string errors;
+	std::auto_ptr<Configuration> pconfig;
+
+	try
+	{
+		pconfig = Configuration::LoadAndVerify(
+			mConfigFileName.c_str(), 
+			GetConfigVerify(), errors);
+	}
+	catch(BoxException &e)
+	{
+		if(e.GetType() == CommonException::ExceptionType &&
+			e.GetSubType() == CommonException::OSFileOpenError)
+		{
+			BOX_ERROR("Failed to open configuration file: " 
+				<< mConfigFileName);
+			return false;
+		}
+
+		throw;
+	}
+
+	// Got errors?
+	if(pconfig.get() == 0 || !errors.empty())
+	{
+		BOX_ERROR("Configuration errors: " << errors);
+		return false;
+	}
+	
+	// Store configuration
+	mpConfiguration = pconfig.release();
+	mLoadedConfigModifiedTime = GetConfigFileModifiedTime();
+	
+	// Let the derived class have a go at setting up stuff
+	// in the initial process
+	SetupInInitialProcess();
+		
+	return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
 //		Name:    Daemon::Main(const std::string& rConfigFileName)
 //		Purpose: Starts the daemon off -- equivalent of C main() function
 //		Created: 2003/07/29
@@ -367,53 +435,17 @@
 
 	std::string pidFileName;
 
-	mConfigFileName = rConfigFileName;
-	
-	bool asDaemon   = !mSingleProcess && !mRunInForeground;
+	bool asDaemon = !mSingleProcess && !mRunInForeground;
 
 	try
 	{
-		// Load the configuration file.
-		std::string errors;
-		std::auto_ptr<Configuration> pconfig;
-
-		try
+		if (!Configure(rConfigFileName))
 		{
-			pconfig = Configuration::LoadAndVerify(
-				mConfigFileName.c_str(), 
-				GetConfigVerify(), errors);
-		}
-		catch(BoxException &e)
-		{
-			if(e.GetType() == CommonException::ExceptionType &&
-				e.GetSubType() == CommonException::OSFileOpenError)
-			{
-				BOX_FATAL("Failed to start: failed to open "
-					"configuration file: " 
-					<< mConfigFileName);
-				return 1;
-			}
-
-			throw;
-		}
-
-		// Got errors?
-		if(pconfig.get() == 0 || !errors.empty())
-		{
-			// Tell user about errors
-			BOX_FATAL("Failed to start: errors in configuration "
-				"file: " << mConfigFileName << ": " << errors);
-			// And give up
+			BOX_FATAL("Failed to start: failed to load "
+				"configuration file: " << rConfigFileName);
 			return 1;
 		}
 		
-		// Store configuration
-		mpConfiguration = pconfig.release();
-		mLoadedConfigModifiedTime = GetConfigFileModifiedTime();
-		
-		// Let the derived class have a go at setting up stuff in the initial process
-		SetupInInitialProcess();
-		
 		// Server configuration
 		const Configuration &serverConfig(
 			mpConfiguration->GetSubConfiguration("Server"));
@@ -675,8 +707,19 @@
 
 #ifdef WIN32
 	WSACleanup();
+#else
+	// Should clean up here, but it breaks memory leak tests.
+	/*
+	if(asDaemon)
+	{
+		// we are running in the child by now, and should not return
+		delete mpConfiguration;
+		mpConfiguration = NULL;
+		exit(0);
+	}
+	*/
 #endif
-	
+
 	return retcode;
 }
 
@@ -840,8 +883,10 @@
 //
 // Function
 //		Name:    Daemon::SetupInInitialProcess()
-//		Purpose: A chance for the daemon to do something initial setting up in the process which
-//				 initiates everything, and after the configuration file has been read and verified.
+//		Purpose: A chance for the daemon to do something initial
+//			 setting up in the process which initiates
+//			 everything, and after the configuration file has
+//			 been read and verified.
 //		Created: 2003/08/20
 //
 // --------------------------------------------------------------------------

Modified: box/trunk/lib/server/Daemon.h
===================================================================
--- box/trunk/lib/server/Daemon.h	2008-05-28 14:27:21 UTC (rev 2176)
+++ box/trunk/lib/server/Daemon.h	2008-05-28 14:28:16 UTC (rev 2177)
@@ -54,7 +54,9 @@
 	virtual std::string DaemonBanner() const;
 	virtual const ConfigurationVerify *GetConfigVerify() const;
 	virtual void Usage();
-	
+
+	virtual bool Configure(const std::string& rConfigFileName);
+
 	bool StopRun() {return mReloadConfigWanted | mTerminateWanted;}
 	bool IsReloadConfigWanted() {return mReloadConfigWanted;}
 	bool IsTerminateWanted() {return mTerminateWanted;}
@@ -63,12 +65,12 @@
 	void SetReloadConfigWanted() {mReloadConfigWanted = true;}
 	void SetTerminateWanted() {mTerminateWanted = true;}
 	
-	virtual void SetupInInitialProcess();
 	virtual void EnterChild();
 	
 	static void SetProcessTitle(const char *format, ...);
 
 protected:
+	virtual void SetupInInitialProcess();
 	box_time_t GetLoadedConfigModifiedTime() const;
 	bool IsSingleProcess() { return mSingleProcess; }
 	virtual std::string GetOptionString();



From boxbackup-dev at fluffy.co.uk  Wed May 28 15:35:20 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 15:35:20 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2178 - box/trunk/lib/common
Message-ID: <20080528143520.C2C9A325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 15:35:20 +0100 (Wed, 28 May 2008)
New Revision: 2178

Modified:
   box/trunk/lib/common/Utils.cpp
Log:
Demangle C++ names in backtrace on GCC using C++ ABI routines.


Modified: box/trunk/lib/common/Utils.cpp
===================================================================
--- box/trunk/lib/common/Utils.cpp	2008-05-28 14:28:16 UTC (rev 2177)
+++ box/trunk/lib/common/Utils.cpp	2008-05-28 14:35:20 UTC (rev 2178)
@@ -18,6 +18,10 @@
 	#include <stdlib.h>
 #endif
 
+#ifdef HAVE_CXXABI_H
+	#include <cxxabi.h>
+#endif
+
 #include "Utils.h"
 #include "CommonException.h"
 #include "Logging.h"
@@ -76,7 +80,67 @@
 
 	for(i = 0; i < size; i++)
 	{
-		BOX_TRACE(strings[i]);
+		// Demangling code copied from 
+		// cctbx_sources/boost_adaptbx/meta_ext.cpp, BSD license
+		
+		std::string mangled_frame = strings[i];
+		std::string output_frame  = strings[i]; // default
+
+		#ifdef HAVE_CXXABI_H
+		int start = mangled_frame.find('(');
+		int end   = mangled_frame.find('+', start);
+		std::string mangled_func = mangled_frame.substr(start + 1,
+			end - start - 1);
+
+		size_t len = 256;
+		std::auto_ptr<char> output_buf(new char [len]);
+		int status;
+		
+		if (abi::__cxa_demangle(mangled_func.c_str(), output_buf.get(),
+			&len, &status) == NULL)
+		{
+			if (status == 0)
+			{
+				BOX_WARNING("Demangle failed but no error: " <<
+					mangled_func);
+			}
+			else if (status == -1)
+			{
+				BOX_WARNING("Demangle failed with "
+					"memory allocation error: " <<
+					mangled_func);
+			}
+			else if (status == -2)
+			{
+				/*
+				BOX_WARNING("Demangle failed with "
+					"with invalid name: " <<
+					mangled_func);
+				*/
+			}
+			else if (status == -3)
+			{
+				BOX_WARNING("Demangle failed with "
+					"with invalid argument: " <<
+					mangled_func);
+			}
+			else
+			{
+				BOX_WARNING("Demangle failed with "
+					"with unknown error " << status <<
+					": " << mangled_func);
+			}
+		}
+		else
+		{
+			output_frame = mangled_frame.substr(0, start + 1) +
+				// std::string(output_buf.get()) +
+				output_buf.get() +
+				mangled_frame.substr(end);
+		}
+		#endif // HAVE_CXXABI_H
+
+		BOX_TRACE("Stack frame " << i << ": " << output_frame);
 	}
 
 #include "MemLeakFindOff.h"



From boxbackup-dev at fluffy.co.uk  Wed May 28 15:37:13 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 15:37:13 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2179 - box/trunk/bin/bbstored
Message-ID: <20080528143713.40CBF325024@www.boxbackup.org>

Author: chris
Date: 2008-05-28 15:37:13 +0100 (Wed, 28 May 2008)
New Revision: 2179

Modified:
   box/trunk/bin/bbstored/BackupCommands.cpp
Log:
Catch exception on reading a directory that doesn't exist, and return a 
protocol error message instead.


Modified: box/trunk/bin/bbstored/BackupCommands.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupCommands.cpp	2008-05-28 14:35:20 UTC (rev 2178)
+++ box/trunk/bin/bbstored/BackupCommands.cpp	2008-05-28 14:37:13 UTC (rev 2179)
@@ -13,20 +13,20 @@
 #include <sstream>
 
 #include "autogen_BackupProtocolServer.h"
+#include "autogen_RaidFileException.h"
 #include "BackupConstants.h"
 #include "BackupContext.h"
-#include "CollectInBufferStream.h"
+#include "BackupStoreConstants.h"
 #include "BackupStoreDirectory.h"
 #include "BackupStoreException.h"
 #include "BackupStoreFile.h"
-#include "StreamableMemBlock.h"
-#include "BackupStoreConstants.h"
-#include "RaidFileController.h"
 #include "BackupStoreInfo.h"
-#include "RaidFileController.h"
+#include "BufferedStream.h"
+#include "CollectInBufferStream.h"
 #include "FileStream.h"
 #include "InvisibleTempFileStream.h"
-#include "BufferedStream.h"
+#include "RaidFileController.h"
+#include "StreamableMemBlock.h"
 
 #include "MemLeakFindOn.h"
 
@@ -181,19 +181,37 @@
 {
 	CHECK_PHASE(Phase_Commands)
 
-	// Ask the context for a directory
-	const BackupStoreDirectory &rdir(rContext.GetDirectory(mObjectID));
-	
 	// Store the listing to a stream
 	std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
-	rdir.WriteToStream(*stream, mFlagsMustBeSet, mFlagsNotToBeSet, mSendAttributes,
-		false /* never send dependency info to the client */);
+
+	try
+	{
+		// Ask the context for a directory
+		const BackupStoreDirectory &rdir(
+			rContext.GetDirectory(mObjectID));
+		rdir.WriteToStream(*stream, mFlagsMustBeSet, 
+			mFlagsNotToBeSet, mSendAttributes,
+			false /* never send dependency info to the client */);
+	}
+	catch (RaidFileException &e)
+	{
+		if (e.GetSubType() == RaidFileException::RaidFileDoesntExist)
+		{
+			return std::auto_ptr<ProtocolObject>(
+				new BackupProtocolServerError(
+					BackupProtocolServerError::ErrorType,
+					BackupProtocolServerError::Err_DoesNotExist));
+		}
+		throw;
+	}
+
 	stream->SetForReading();
 	
 	// Get the protocol to send the stream
 	rProtocol.SendStreamAfterCommand(stream.release());
 
-	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+	return std::auto_ptr<ProtocolObject>(
+		new BackupProtocolServerSuccess(mObjectID));
 }
 
 // --------------------------------------------------------------------------



From boxbackup-dev at fluffy.co.uk  Wed May 28 15:40:39 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 15:40:39 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2180 - in box/trunk/bin: bbackupd bbstored
Message-ID: <20080528144039.E3003325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 15:40:39 +0100 (Wed, 28 May 2008)
New Revision: 2180

Modified:
   box/trunk/bin/bbackupd/bbackupd.cpp
   box/trunk/bin/bbstored/bbstored.cpp
Log:
Shorten names used by daemons.


Modified: box/trunk/bin/bbackupd/bbackupd.cpp
===================================================================
--- box/trunk/bin/bbackupd/bbackupd.cpp	2008-05-28 14:37:13 UTC (rev 2179)
+++ box/trunk/bin/bbackupd/bbackupd.cpp	2008-05-28 14:40:39 UTC (rev 2180)
@@ -29,7 +29,7 @@
 
 	MAINHELPER_START
 
-	Logging::SetProgramName("Box Backup (bbackupd)");
+	Logging::SetProgramName("bbackupd");
 	Logging::ToConsole(true);
 	Logging::ToSyslog (true);
 	

Modified: box/trunk/bin/bbstored/bbstored.cpp
===================================================================
--- box/trunk/bin/bbstored/bbstored.cpp	2008-05-28 14:37:13 UTC (rev 2179)
+++ box/trunk/bin/bbstored/bbstored.cpp	2008-05-28 14:40:39 UTC (rev 2180)
@@ -18,7 +18,7 @@
 {
 	MAINHELPER_START
 
-	Logging::SetProgramName("Box Backup (bbstored)");
+	Logging::SetProgramName("bbstored");
 	Logging::ToConsole(true);
 	Logging::ToSyslog (true);
 



From boxbackup-dev at fluffy.co.uk  Wed May 28 16:24:05 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 16:24:05 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2181 - box/trunk/bin/bbackupd
Message-ID: <20080528152405.C3808325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 16:24:05 +0100 (Wed, 28 May 2008)
New Revision: 2181

Modified:
   box/trunk/bin/bbackupd/BackupClientContext.cpp
   box/trunk/bin/bbackupd/BackupClientContext.h
   box/trunk/bin/bbackupd/BackupClientDeleteList.cpp
   box/trunk/bin/bbackupd/BackupClientDeleteList.h
   box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
   box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h
   box/trunk/bin/bbackupd/BackupDaemon.cpp
   box/trunk/bin/bbackupd/BackupDaemon.h
Log:
Track and log file deletions by name.

Split crypto init and file sync process into its own method, to reduce 
call depth and facilitate calling in process from tests.

Differentiate between 3 uses of stat in BackupClientDirectoryRecord by 
renaming the structures.

Use stat instead of lstat when checking the filesystem that's holding an 
entity, in case it's a symbolic link to a different filesystem.


Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientContext.cpp	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupClientContext.cpp	2008-05-28 15:24:05 UTC (rev 2181)
@@ -48,7 +48,8 @@
 	int32_t AccountNumber, 
 	bool ExtendedLogging,
 	bool ExtendedLogToFile,
-	std::string ExtendedLogFile
+	std::string ExtendedLogFile,
+	ProgressNotifier& rProgressNotifier
 )
 	: mrDaemon(rDaemon),
 	  mrTLSContext(rTLSContext),
@@ -69,7 +70,8 @@
 	  mpExcludeFiles(0),
 	  mpExcludeDirs(0),
 	  mKeepAliveTimer(0),
-	  mbIsManaged(false)
+	  mbIsManaged(false),
+	  mrProgressNotifier(rProgressNotifier)
 {
 }
 

Modified: box/trunk/bin/bbackupd/BackupClientContext.h
===================================================================
--- box/trunk/bin/bbackupd/BackupClientContext.h	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupClientContext.h	2008-05-28 15:24:05 UTC (rev 2181)
@@ -12,6 +12,7 @@
 
 #include "BoxTime.h"
 #include "BackupClientDeleteList.h"
+#include "BackupClientDirectoryRecord.h"
 #include "BackupStoreFile.h"
 #include "ExcludeList.h"
 #include "Timer.h"
@@ -45,7 +46,8 @@
 		int32_t AccountNumber, 
 		bool ExtendedLogging,
 		bool ExtendedLogToFile,
-		std::string ExtendedLogFile
+		std::string ExtendedLogFile,
+		ProgressNotifier &rProgressNotifier
 	);
 	virtual ~BackupClientContext();
 private:
@@ -70,6 +72,7 @@
 	int64_t GetClientStoreMarker() const {return mClientStoreMarker;}
 	
 	bool StorageLimitExceeded() {return mStorageLimitExceeded;}
+	void SetStorageLimitExceeded() {mStorageLimitExceeded = true;}
 
 	// --------------------------------------------------------------------------
 	//
@@ -198,6 +201,11 @@
 	virtual int    GetMaximumDiffingTime();
 	virtual bool   IsManaged() { return mbIsManaged; }
 	
+	ProgressNotifier& GetProgressNotifier() const 
+	{ 
+		return mrProgressNotifier;
+	}
+
 private:
 	BackupDaemon &mrDaemon;
 	TLSContext &mrTLSContext;
@@ -221,6 +229,7 @@
 	bool mbIsManaged;
 	int mKeepAliveTime;
 	int mMaximumDiffingTime;
+	ProgressNotifier &mrProgressNotifier;
 };
 
 #endif // BACKUPCLIENTCONTEXT__H

Modified: box/trunk/bin/bbackupd/BackupClientDeleteList.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDeleteList.cpp	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupClientDeleteList.cpp	2008-05-28 15:24:05 UTC (rev 2181)
@@ -42,21 +42,38 @@
 {
 }
 
+BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID,
+	const BackupStoreFilename& rFilename,
+	const std::string& rLocalPath)
+: mDirectoryID(DirectoryID),
+  mFilename(rFilename),
+  mLocalPath(rLocalPath)
+{ }
+
+BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID,
+	const std::string& rLocalPath)
+: mObjectID(ObjectID),
+  mLocalPath(rLocalPath)
+{ }
+
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupClientDeleteList::AddDirectoryDelete(int64_t)
+//		Name:    BackupClientDeleteList::AddDirectoryDelete(int64_t,
+//			 const BackupStoreFilename&)
 //		Purpose: Add a directory to the list of directories to be deleted.
 //		Created: 10/11/03
 //
 // --------------------------------------------------------------------------
-void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID)
+void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID,
+	const std::string& rLocalPath)
 {
 	// Only add the delete to the list if it's not in the "no delete" set
-	if(mDirectoryNoDeleteList.find(ObjectID) == mDirectoryNoDeleteList.end())
+	if(mDirectoryNoDeleteList.find(ObjectID) ==
+		mDirectoryNoDeleteList.end())
 	{
 		// Not in the list, so should delete it
-		mDirectoryList.push_back(ObjectID);
+		mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath));
 	}
 }
 
@@ -64,18 +81,22 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupClientDeleteList::AddFileDelete(int64_t, BackupStoreFilenameClear &)
+//		Name:    BackupClientDeleteList::AddFileDelete(int64_t,
+//			 const BackupStoreFilename &)
 //		Purpose: 
 //		Created: 10/11/03
 //
 // --------------------------------------------------------------------------
-void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename)
+void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID,
+	const BackupStoreFilename &rFilename, const std::string& rLocalPath)
 {
 	// Try to find it in the no delete list
-	std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator delEntry(mFileNoDeleteList.begin());
+	std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator
+		delEntry(mFileNoDeleteList.begin());
 	while(delEntry != mFileNoDeleteList.end())
 	{
-		if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename)
+		if((delEntry)->first == DirectoryID 
+			&& (delEntry)->second == rFilename)
 		{
 			// Found!
 			break;
@@ -86,7 +107,8 @@
 	// Only add it to the delete list if it wasn't in the no delete list
 	if(delEntry == mFileNoDeleteList.end())
 	{
-		mFileList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename));
+		mFileList.push_back(FileToDelete(DirectoryID, rFilename,
+			rLocalPath));
 	}
 }
 
@@ -113,18 +135,24 @@
 	BackupProtocolClient &connection(rContext.GetConnection());
 	
 	// Do the deletes
-	for(std::vector<int64_t>::iterator i(mDirectoryList.begin()); i != mDirectoryList.end(); ++i)
+	for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin());
+		i != mDirectoryList.end(); ++i)
 	{
-		connection.QueryDeleteDirectory(*i);
+		connection.QueryDeleteDirectory(i->mObjectID);
+		rContext.GetProgressNotifier().NotifyDirectoryDeleted(
+			i->mObjectID, i->mLocalPath);
 	}
 	
 	// Clear the directory list
 	mDirectoryList.clear();
 	
 	// Delete the files
-	for(std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator i(mFileList.begin()); i != mFileList.end(); ++i)
+	for(std::vector<FileToDelete>::iterator i(mFileList.begin());
+		i != mFileList.end(); ++i)
 	{
-		connection.QueryDeleteFile(i->first, i->second);
+		connection.QueryDeleteFile(i->mDirectoryID, i->mFilename);
+		rContext.GetProgressNotifier().NotifyFileDeleted(
+			i->mDirectoryID, i->mLocalPath);
 	}
 }
 
@@ -140,7 +168,15 @@
 void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID)
 {
 	// First of all, is it in the delete vector?
-	std::vector<int64_t>::iterator delEntry(std::find(mDirectoryList.begin(), mDirectoryList.end(), ObjectID));
+	std::vector<DirToDelete>::iterator delEntry(mDirectoryList.begin());
+	for(; delEntry != mDirectoryList.end(); delEntry++)
+	{
+		if(delEntry->mObjectID == ObjectID)
+		{
+			// Found!
+			break;
+		}
+	}
 	if(delEntry != mDirectoryList.end())
 	{
 		// erase this entry
@@ -148,7 +184,8 @@
 	}
 	else
 	{
-		// Haven't been asked to delete it yet, put it in the no delete list
+		// Haven't been asked to delete it yet, put it in the
+		// no delete list
 		mDirectoryNoDeleteList.insert(ObjectID);
 	}
 }
@@ -162,13 +199,15 @@
 //		Created: 19/11/03
 //
 // --------------------------------------------------------------------------
-void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename)
+void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID,
+	const BackupStoreFilename &rFilename)
 {
 	// Find this in the delete list
-	std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator delEntry(mFileList.begin());
+	std::vector<FileToDelete>::iterator delEntry(mFileList.begin());
 	while(delEntry != mFileList.end())
 	{
-		if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename)
+		if(delEntry->mDirectoryID == DirectoryID
+			&& delEntry->mFilename == rFilename)
 		{
 			// Found!
 			break;
@@ -186,10 +225,5 @@
 		// Haven't been asked to delete it yet, put it in the no delete list
 		mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename));
 	}
-
 }
 
-
-
-
-

Modified: box/trunk/bin/bbackupd/BackupClientDeleteList.h
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDeleteList.h	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupClientDeleteList.h	2008-05-28 15:24:05 UTC (rev 2181)
@@ -28,22 +28,46 @@
 // --------------------------------------------------------------------------
 class BackupClientDeleteList
 {
+private:
+	class FileToDelete
+	{
+		public:
+		int64_t mDirectoryID;
+		BackupStoreFilename mFilename;
+		std::string mLocalPath;
+		FileToDelete(int64_t DirectoryID, 
+			const BackupStoreFilename& rFilename,
+			const std::string& rLocalPath);
+	};
+
+	class DirToDelete
+	{
+		public:
+		int64_t mObjectID;
+		std::string mLocalPath;
+		DirToDelete(int64_t ObjectID, const std::string& rLocalPath);
+	};
+
 public:
 	BackupClientDeleteList();
 	~BackupClientDeleteList();
 	
-	void AddDirectoryDelete(int64_t ObjectID);
-	void AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename);
+	void AddDirectoryDelete(int64_t ObjectID,
+		const std::string& rLocalPath);
+	void AddFileDelete(int64_t DirectoryID,
+		const BackupStoreFilename &rFilename,
+		const std::string& rLocalPath);
 
 	void StopDirectoryDeletion(int64_t ObjectID);
-	void StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename);
+	void StopFileDeletion(int64_t DirectoryID,
+		const BackupStoreFilename &rFilename);
 	
 	void PerformDeletions(BackupClientContext &rContext);
 	
 private:
-	std::vector<int64_t> mDirectoryList;
+	std::vector<DirToDelete> mDirectoryList;
 	std::set<int64_t> mDirectoryNoDeleteList;	// note: things only get in this list if they're not present in mDirectoryList when they are 'added'
-	std::vector<std::pair<int64_t, BackupStoreFilename> > mFileList;
+	std::vector<FileToDelete> mFileList;
 	std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList;
 };
 

Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp	2008-05-28 15:24:05 UTC (rev 2181)
@@ -2,7 +2,8 @@
 //
 // File
 //		Name:    BackupClientDirectoryRecord.cpp
-//		Purpose: Implementation of record about directory for backup client
+//		Purpose: Implementation of record about directory for
+//			 backup client
 //		Created: 2003/10/08
 //
 // --------------------------------------------------------------------------
@@ -100,14 +101,25 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool)
-//		Purpose: Syncronise, recusively, a local directory with the server.
+//		Name:    BackupClientDirectoryRecord::SyncDirectory(i
+//			 BackupClientDirectoryRecord::SyncParams &,
+//			 int64_t, const std::string &,
+//			 const std::string &, bool)
+//		Purpose: Recursively synchronise a local directory
+//			 with the server.
 //		Created: 2003/10/08
 //
 // --------------------------------------------------------------------------
-void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &rParams, int64_t ContainingDirectoryID,
-	const std::string &rLocalPath, bool ThisDirHasJustBeenCreated)
+void BackupClientDirectoryRecord::SyncDirectory(
+	BackupClientDirectoryRecord::SyncParams &rParams,
+	int64_t ContainingDirectoryID,
+	const std::string &rLocalPath,
+	const std::string &rRemotePath,
+	bool ThisDirHasJustBeenCreated)
 {
+	BackupClientContext& rContext(rParams.mrContext);
+	ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
 	// Signal received by daemon?
 	if(rParams.mrDaemon.StopRun())
 	{
@@ -118,49 +130,66 @@
 	// Start by making some flag changes, marking this sync as not done,
 	// and on the immediate sub directories.
 	mSyncDone = false;
-	for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin();
+	for(std::map<std::string, BackupClientDirectoryRecord *>::iterator
+		i  = mSubDirectories.begin();
 		i != mSubDirectories.end(); ++i)
 	{
 		i->second->mSyncDone = false;
 	}
 
-	// Work out the time in the future after which the file should be uploaded regardless.
-	// This is a simple way to avoid having too many problems with file servers when they have
-	// clients with badly out of sync clocks.
-	rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + rParams.mMaxFileTimeInFuture;
+	// Work out the time in the future after which the file should
+	// be uploaded regardless. This is a simple way to avoid having
+	// too many problems with file servers when they have clients
+	// with badly out of sync clocks.
+	rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() +
+		rParams.mMaxFileTimeInFuture;
 	
-	// Build the current state checksum to compare against while getting info from dirs
-	// Note checksum is used locally only, so byte order isn't considered.
+	// Build the current state checksum to compare against while
+	// getting info from dirs. Note checksum is used locally only,
+	// so byte order isn't considered.
 	MD5Digest currentStateChecksum;
 	
+	struct stat dest_st;
 	// Stat the directory, to get attribute info
+	// If it's a symbolic link, we want the link target here
+	// (as we're about to back up the contents of the directory)
 	{
-		struct stat st;
-		if(::stat(rLocalPath.c_str(), &st) != 0)
+		if(::stat(rLocalPath.c_str(), &dest_st) != 0)
 		{
-			// The directory has probably been deleted, so just ignore this error.
-			// In a future scan, this deletion will be noticed, deleted from server, and this object deleted.
-			rParams.GetProgressNotifier().NotifyDirStatFailed(
-				this, rLocalPath, strerror(errno));
+			// The directory has probably been deleted, so
+			// just ignore this error. In a future scan, this
+			// deletion will be noticed, deleted from server,
+			// and this object deleted.
+			rNotifier.NotifyDirStatFailed(this, rLocalPath,
+				strerror(errno));
 			return;
 		}
-		// Store inode number in map so directories are tracked in case they're renamed
+		// Store inode number in map so directories are tracked
+		// in case they're renamed
 		{
-			BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
-			idMap.AddToMap(st.st_ino, mObjectID, ContainingDirectoryID);
+			BackupClientInodeToIDMap &idMap(
+				rParams.mrContext.GetNewIDMap());
+			idMap.AddToMap(dest_st.st_ino, mObjectID,
+				ContainingDirectoryID);
 		}
 		// Add attributes to checksum
-		currentStateChecksum.Add(&st.st_mode, sizeof(st.st_mode));
-		currentStateChecksum.Add(&st.st_uid, sizeof(st.st_uid));
-		currentStateChecksum.Add(&st.st_gid, sizeof(st.st_gid));
+		currentStateChecksum.Add(&dest_st.st_mode,
+			sizeof(dest_st.st_mode));
+		currentStateChecksum.Add(&dest_st.st_uid,
+			sizeof(dest_st.st_uid));
+		currentStateChecksum.Add(&dest_st.st_gid,
+			sizeof(dest_st.st_gid));
 		// Inode to be paranoid about things moving around
-		currentStateChecksum.Add(&st.st_ino, sizeof(st.st_ino));
+		currentStateChecksum.Add(&dest_st.st_ino,
+			sizeof(dest_st.st_ino));
 #ifdef HAVE_STRUCT_STAT_ST_FLAGS
-		currentStateChecksum.Add(&st.st_flags, sizeof(st.st_flags));
+		currentStateChecksum.Add(&dest_st.st_flags,
+			sizeof(dest_st.st_flags));
 #endif
 
 		StreamableMemBlock xattr;
-		BackupClientFileAttributes::FillExtendedAttr(xattr, rLocalPath.c_str());
+		BackupClientFileAttributes::FillExtendedAttr(xattr,
+			rLocalPath.c_str());
 		currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize());
 	}
 	
@@ -170,13 +199,13 @@
 	std::vector<std::string> files;
 	bool downloadDirectoryRecordBecauseOfFutureFiles = false;
 
-	struct stat dir_st;
-	if(::lstat(rLocalPath.c_str(), &dir_st) != 0)
+	struct stat link_st;
+	if(::lstat(rLocalPath.c_str(), &link_st) != 0)
 	{
 		// Report the error (logs and 
 		// eventual email to administrator)
-		rParams.GetProgressNotifier().NotifyFileStatFailed(this, 
-			rLocalPath, strerror(errno));
+		rNotifier.NotifyFileStatFailed(this, rLocalPath,
+			strerror(errno));
 		
 		// FIXME move to NotifyFileStatFailed()
 		SetErrorWhenReadingFilesystemObject(rParams, 
@@ -192,8 +221,7 @@
 		DIR *dirHandle = 0;
 		try
 		{
-			rParams.GetProgressNotifier().NotifyScanDirectory(
-				this, rLocalPath);
+			rNotifier.NotifyScanDirectory(this, rLocalPath);
 
 			dirHandle = ::opendir(rLocalPath.c_str());
 			if(dirHandle == 0)
@@ -202,17 +230,19 @@
 				// eventual email to administrator)
 				if (errno == EACCES)
 				{
-					rParams.GetProgressNotifier().NotifyDirListFailed(
-						this, rLocalPath, "Access denied");
+					rNotifier.NotifyDirListFailed(this,
+						rLocalPath, "Access denied");
 				}
 				else
 				{
-					rParams.GetProgressNotifier().NotifyDirListFailed(this, 
+					rNotifier.NotifyDirListFailed(this, 
 						rLocalPath, strerror(errno));
 				}
 				
-				// Report the error (logs and eventual email to administrator)
-				SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str());
+				// Report the error (logs and eventual email
+				// to administrator)
+				SetErrorWhenReadingFilesystemObject(rParams,
+					rLocalPath.c_str());
 				// Ignore this directory for now.
 				return;
 			}
@@ -228,14 +258,17 @@
 			::memset(&checksum_info, 0, sizeof(checksum_info));
 	
 			struct dirent *en = 0;
-			struct stat st;
+			struct stat file_st;
 			std::string filename;
 			while((en = ::readdir(dirHandle)) != 0)
 			{
 				rParams.mrContext.DoKeepAlive();
 				
-				// Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str());
-				// on Linux, as a stat is performed to get all this info
+				// Don't need to use
+				// LinuxWorkaround_FinishDirentStruct(en,
+				// rLocalPath.c_str());
+				// on Linux, as a stat is performed to
+				// get all this info
 
 				if(en->d_name[0] == '.' && 
 					(en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0')))
@@ -259,11 +292,11 @@
 				// prefer S_IFREG, S_IFDIR...
 				int type = en->d_type;
 				#else
-				if(::lstat(filename.c_str(), &st) != 0)
+				if(::lstat(filename.c_str(), &file_st) != 0)
 				{
 					// Report the error (logs and 
 					// eventual email to administrator)
- 					rParams.GetProgressNotifier().NotifyFileStatFailed(this, 
+ 					rNotifier.NotifyFileStatFailed(this, 
  						filename, strerror(errno));
 					
 					// FIXME move to NotifyFileStatFailed()
@@ -274,19 +307,18 @@
 					continue;
 				}
 
-				if(st.st_dev != dir_st.st_dev)
+				if(file_st.st_dev != dest_st.st_dev)
 				{
 					if(!(rParams.mrContext.ExcludeDir(
 						filename)))
 					{
-						rParams.GetProgressNotifier()
-							.NotifyMountPointSkipped(
-								this, filename);
+						rNotifier.NotifyMountPointSkipped(
+							this, filename);
 					}
 					continue;
 				}
 
-				int type = st.st_mode & S_IFMT;
+				int type = file_st.st_mode & S_IFMT;
 				#endif
 
 				if(type == S_IFREG || type == S_IFLNK)
@@ -296,8 +328,7 @@
 					// Exclude it?
 					if(rParams.mrContext.ExcludeFile(filename))
 					{
- 						rParams.GetProgressNotifier()
-							.NotifyFileExcluded(
+ 						rNotifier.NotifyFileExcluded(
 								this, 
 								filename);
 
@@ -315,8 +346,7 @@
 					// Exclude it?
 					if(rParams.mrContext.ExcludeDir(filename))
 					{
- 						rParams.GetProgressNotifier()
-							.NotifyDirExcluded(
+ 						rNotifier.NotifyDirExcluded(
 								this, 
 								filename);
 
@@ -331,15 +361,13 @@
 				{
 					if(rParams.mrContext.ExcludeFile(filename))
 					{
- 						rParams.GetProgressNotifier()
-							.NotifyFileExcluded(
+ 						rNotifier.NotifyFileExcluded(
 								this, 
 								filename);
 					}
 					else
 					{
- 						rParams.GetProgressNotifier()
-							.NotifyUnsupportedFileType(
+ 						rNotifier.NotifyUnsupportedFileType(
 								this, filename);
 						SetErrorWhenReadingFilesystemObject(
 							rParams, filename.c_str());
@@ -354,10 +382,9 @@
 				#ifdef WIN32
 				// We didn't stat the file before,
 				// but now we need the information.
-				if(::lstat(filename.c_str(), &st) != 0)
+				if(::lstat(filename.c_str(), &file_st) != 0)
 				{
- 					rParams.GetProgressNotifier()
-						.NotifyFileStatFailed(this, 
+ 					rNotifier.NotifyFileStatFailed(this, 
  							filename, 
 							strerror(errno));
 					
@@ -370,18 +397,17 @@
 					continue;
 				}
 
-				if(st.st_dev != dir_st.st_dev)
+				if(file_st.st_dev != link_st.st_dev)
 				{
- 					rParams.GetProgressNotifier()
-						.NotifyMountPointSkipped(this, 
+ 					rNotifier.NotifyMountPointSkipped(this, 
  							filename);
 					continue;
 				}
 				#endif
 
-				checksum_info.mModificationTime = FileModificationTime(st);
-				checksum_info.mAttributeModificationTime = FileAttrModificationTime(st);
-				checksum_info.mSize = st.st_size;
+				checksum_info.mModificationTime = FileModificationTime(file_st);
+				checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st);
+				checksum_info.mSize = file_st.st_size;
 				currentStateChecksum.Add(&checksum_info, sizeof(checksum_info));
 				currentStateChecksum.Add(en->d_name, strlen(en->d_name));
 				
@@ -394,7 +420,7 @@
 					// Log that this has happened
 					if(!rParams.mHaveLoggedWarningAboutFutureFileTimes)
 					{
-						rParams.GetProgressNotifier().NotifyFileModifiedInFuture(
+						rNotifier.NotifyFileModifiedInFuture(
 							this, filename);
 						rParams.mHaveLoggedWarningAboutFutureFileTimes = true;
 					}
@@ -468,7 +494,8 @@
 		}
 		
 		// Do the directory reading
-		bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, pdirOnStore, entriesLeftOver, files, dirs);
+		bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath,
+			rRemotePath, pdirOnStore, entriesLeftOver, files, dirs);
 		
 		// LAST THING! (think exception safety)
 		// Store the new checksum -- don't fetch things unnecessarily in the future
@@ -604,11 +631,18 @@
 //		Created: 2003/10/09
 //
 // --------------------------------------------------------------------------
-bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &rParams,
-	const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+bool BackupClientDirectoryRecord::UpdateItems(
+	BackupClientDirectoryRecord::SyncParams &rParams,
+	const std::string &rLocalPath,
+	const std::string &rRemotePath,
+	BackupStoreDirectory *pDirOnStore,
 	std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
-	std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs)
+	std::vector<std::string> &rFiles,
+	const std::vector<std::string> &rDirs)
 {
+	BackupClientContext& rContext(rParams.mrContext);
+	ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
 	bool allUpdatedSuccessfully = true;
 
 	// Decrypt all the directory entries.
@@ -634,7 +668,7 @@
 		f != rFiles.end(); ++f)
 	{
 		// Send keep-alive message if needed
-		rParams.mrContext.DoKeepAlive();
+		rContext.DoKeepAlive();
 		
 		// Filename of this file
 		std::string filename(MakeFullPath(rLocalPath, *f));
@@ -651,7 +685,7 @@
 			struct stat st;
 			if(::lstat(filename.c_str(), &st) != 0)
 			{
-				rParams.GetProgressNotifier().NotifyFileStatFailed(this, 
+				rNotifier.NotifyFileStatFailed(this, 
 					filename, strerror(errno));
 
 				// Report the error (logs and 
@@ -689,7 +723,8 @@
 		if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0))
 		{
 			// Directory exists in the place of this file -- sort it out
-			RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, en->GetObjectID(), *f);
+			RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore,
+				en, *f);
 			en = 0;
 		}
 		
@@ -701,7 +736,7 @@
 			// 2) It's not in the store
 			
 			// Do we know about the inode number?
-			const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+			const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap());
 			int64_t renameObjectID = 0, renameInDirectory = 0;
 			if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
 			{
@@ -711,7 +746,7 @@
 				bool isCurrentVersion = false;
 				box_time_t srvModTime = 0, srvAttributesHash = 0;
 				BackupStoreFilenameClear oldLeafname;
-				if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
+				if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
 				{	
 					// Only interested if it's a file and the latest version
 					if(!isDir && isCurrentVersion)
@@ -724,11 +759,11 @@
 							// Therefore we can safely rename it to this new file.
 
 							// Get the connection to the server 
-							BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+							BackupProtocolClient &connection(rContext.GetConnection());
 
 							// Only do this step if there is room on the server.
 							// This step will be repeated later when there is space available
-							if(!rParams.mrContext.StorageLimitExceeded())
+							if(!rContext.StorageLimitExceeded())
 							{
 								// Rename the existing files (ie include old versions) on the server
 								connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */,
@@ -736,7 +771,7 @@
 									storeFilename);
 									
 								// Stop the attempt to delete the file in the original location
-								BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+								BackupClientDeleteList &rdelList(rContext.GetDeleteList());
 								rdelList.StopFileDeletion(renameInDirectory, oldLeafname);
 								
 								// Create new entry in the directory for it
@@ -871,16 +906,22 @@
 				<< rParams.mSyncPeriodEnd << ")");
 		}
 
+		bool fileSynced = true;
+
 		if (doUpload)
 		{
+			// Upload needed, don't mark sync success until
+			// we've actually done it
+			fileSynced = false;
+
 			// Make sure we're connected -- must connect here so we know whether
 			// the storage limit has been exceeded, and hence whether or not
 			// to actually upload the file.
-			rParams.mrContext.GetConnection();
+			rContext.GetConnection();
 
 			// Only do this step if there is room on the server.
 			// This step will be repeated later when there is space available
-			if(!rParams.mrContext.StorageLimitExceeded())
+			if(!rContext.StorageLimitExceeded())
 			{
 				// Upload the file to the server, recording the object ID it returns
 				bool noPreviousVersionOnServer = ((pDirOnStore != 0) && (en == 0));
@@ -890,15 +931,27 @@
 				try
 				{
 					latestObjectID = UploadFile(rParams, filename, storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer);
-					uploadSuccess = true;
+					if (latestObjectID == 0)
+					{
+						// storage limit exceeded
+						rParams.mrContext.SetStorageLimitExceeded();
+						uploadSuccess = false;
+						allUpdatedSuccessfully = false;
+					}
+					else
+					{
+						uploadSuccess = true;
+					}
 				}
 				catch(ConnectionException &e)
 				{
 					// Connection errors should just be passed on to the main handler, retries
 					// would probably just cause more problems.
-					rParams.GetProgressNotifier()
-						.NotifyFileUploadException(
-							this, filename, e);
+					// StorageLimitExceeded never gets here.
+					
+					rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull);
+					rNotifier.NotifyFileUploadException(
+						this, filename, e);
 					throw;
 				}
 				catch(BoxException &e)
@@ -907,14 +960,15 @@
 					allUpdatedSuccessfully = false;
 					// Log it.
 					SetErrorWhenReadingFilesystemObject(rParams, filename.c_str());
-					rParams.GetProgressNotifier()
-						.NotifyFileUploadException(
-							this, filename, e);
+					rNotifier.NotifyFileUploadException(
+						this, filename, e);
 				}
 
 				// Update structures if the file was uploaded successfully.
 				if(uploadSuccess)
 				{
+					fileSynced = true;
+
 					// delete from pending entries
 					if(pendingFirstSeenTime != 0 && mpPendingEntries != 0)
 					{
@@ -924,28 +978,31 @@
 			}
 			else
 			{
-				rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this,
+				rNotifier.NotifyFileSkippedServerFull(this,
 					filename);
 			}
 		}
 		else if(en != 0 && en->GetAttributesHash() != attributesHash)
 		{
 			// Attributes have probably changed, upload them again.
-			// If the attributes have changed enough, the directory hash will have changed too,
-			// and so the dir will have been downloaded, and the entry will be available.
+			// If the attributes have changed enough, the directory
+			// hash will have changed too, and so the dir will have
+			// been downloaded, and the entry will be available.
 
 			// Get connection
-			BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+			BackupProtocolClient &connection(rContext.GetConnection());
 
 			// Only do this step if there is room on the server.
-			// This step will be repeated later when there is space available
-			if(!rParams.mrContext.StorageLimitExceeded())
+			// This step will be repeated later when there is
+			// space available
+			if(!rContext.StorageLimitExceeded())
 			{
 				// Update store
 				BackupClientFileAttributes attr;
 				attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */);
 				MemBlockStream attrStream(attr);
 				connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream);
+				fileSynced = true;
 			}
 		}
 
@@ -981,7 +1038,7 @@
 		if(fileSize >= rParams.mFileTrackingSizeThreshold)
 		{
 			// Get the map
-			BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
+			BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap());
 		
 			// Need to get an ID from somewhere...
 			if(latestObjectID != 0)
@@ -993,7 +1050,7 @@
 			{
 				// Don't know it -- haven't sent anything to the store, and didn't get a listing.
 				// Look it up in the current map, and if it's there, use that.
-				const BackupClientInodeToIDMap &currentIDMap(rParams.mrContext.GetCurrentIDMap());
+				const BackupClientInodeToIDMap &currentIDMap(rContext.GetCurrentIDMap());
 				int64_t objid = 0, dirid = 0;
 				if(currentIDMap.Lookup(inodeNum, objid, dirid))
 				{
@@ -1002,15 +1059,18 @@
 					// NOTE: If the above assert fails, an inode number has been reused by the OS,
 					// or there is a problem somewhere. If this happened on a short test run, look
 					// into it. However, in a long running process this may happen occasionally and
-					// not indiciate anything wrong.
+					// not indicate anything wrong.
 					// Run the release version for real life use, where this check is not made.
 					idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */);				
 				}
 			}
 		}
 		
-		rParams.GetProgressNotifier().NotifyFileSynchronised(this, 
-			filename, fileSize);
+		if (fileSynced)
+		{
+			rNotifier.NotifyFileSynchronised(this, filename,
+				fileSize);
+		}
 	}
 
 	// Erase contents of files to save space when recursing
@@ -1030,7 +1090,7 @@
 		d != rDirs.end(); ++d)
 	{
 		// Send keep-alive message if needed
-		rParams.mrContext.DoKeepAlive();
+		rContext.DoKeepAlive();
 		
 		// Get the local filename
 		std::string dirname(MakeFullPath(rLocalPath, *d));
@@ -1050,16 +1110,20 @@
 		// Check that the entry which might have been found is in fact a directory
 		if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0))
 		{
-			// Entry exists, but is not a directory. Bad. Get rid of it.
-			BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+			// Entry exists, but is not a directory. Bad.
+			// Get rid of it.
+			BackupProtocolClient &connection(rContext.GetConnection());
 			connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename);
+			rNotifier.NotifyFileDeleted(en->GetObjectID(),
+				storeFilename.GetClearFilename());
 			
 			// Nothing found
 			en = 0;
 		}
 
-		// Flag for having created directory, so can optimise the recusive call not to
-		// read it again, because we know it's empty.
+		// Flag for having created directory, so can optimise the
+		// recusive call not to read it again, because we know
+		// it's empty.
 		bool haveJustCreatedDirOnServer = false;
 
 		// Next, see if it's in the list of sub directories
@@ -1086,7 +1150,7 @@
 				// No. Exists on the server, and we know about it from the listing.
 				subDirObjectID = en->GetObjectID();
 			}
-			else if(rParams.mrContext.StorageLimitExceeded())	
+			else if(rContext.StorageLimitExceeded())	
 			// know we've got a connection if we get this far,
 			// as dir will have been modified.
 			{
@@ -1112,14 +1176,15 @@
 				// First, do we have a record in the ID map?
 				int64_t renameObjectID = 0, renameInDirectory = 0;
 				bool renameDir = false;
-				const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+				const BackupClientInodeToIDMap &idMap(
+					rContext.GetCurrentIDMap());
 				if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
 				{
 					// Look up on the server to get the name, to build the local filename
 					std::string localPotentialOldName;
 					bool isDir = false;
 					bool isCurrentVersion = false;
-					if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
+					if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
 					{	
 						// Only interested if it's a directory
 						if(isDir && isCurrentVersion)
@@ -1137,7 +1202,7 @@
 				}
 
 				// Get connection
-				BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+				BackupProtocolClient &connection(rContext.GetConnection());
 				
 				// Don't do a check for storage limit exceeded here, because if we get to this
 				// stage, a connection will have been opened, and the status known, so the check 
@@ -1157,7 +1222,8 @@
 					connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream);
 
 					// Stop it being deleted later
-					BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+					BackupClientDeleteList &rdelList(
+						rContext.GetDeleteList());
 					rdelList.StopDirectoryDeletion(renameObjectID);
 
 					// This is the ID for the renamed directory
@@ -1194,12 +1260,14 @@
 			}
 		}
 		
-		ASSERT(psubDirRecord != 0 || rParams.mrContext.StorageLimitExceeded());
+		ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded());
 		
 		if(psubDirRecord)
 		{
 			// Sync this sub directory too
-			psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, haveJustCreatedDirOnServer);
+			psubDirRecord->SyncDirectory(rParams, mObjectID,
+				dirname, rRemotePath + "/" + *d,
+				haveJustCreatedDirOnServer);
 		}
 
 		// Zero pointer in rEntriesLeftOver, if we have a pointer to zero
@@ -1228,20 +1296,26 @@
 			// to a list, which is actually deleted at the very end of the session.
 			// If there's an error during the process, it doesn't matter if things
 			// aren't actually deleted, as the whole state will be reset anyway.
-			BackupClientDeleteList &rdel(rParams.mrContext.GetDeleteList());
+			BackupClientDeleteList &rdel(rContext.GetDeleteList());
+
+			std::string localName = MakeFullPath(rLocalPath,
+				en->GetName());
 			
 			// Delete this entry -- file or directory?
 			if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
 			{
 				// Set a pending deletion for the file
-				rdel.AddFileDelete(mObjectID, en->GetName());				
+				rdel.AddFileDelete(mObjectID, en->GetName(),
+					localName);
 			}
 			else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0)
 			{
 				// Set as a pending deletion for the directory
-				rdel.AddDirectoryDelete(en->GetObjectID());
+				rdel.AddDirectoryDelete(en->GetObjectID(),
+					localName);
 				
-				// If there's a directory record for it in the sub directory map, delete it now
+				// If there's a directory record for it in 
+				// the sub directory map, delete it now
 				BackupStoreFilenameClear dirname(en->GetName());
 				std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(dirname.GetClearFilename()));
 				if(e != mSubDirectories.end())
@@ -1276,14 +1350,24 @@
 //		Created: 9/7/04
 //
 // --------------------------------------------------------------------------
-void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename)
+void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(
+	SyncParams &rParams,
+	BackupStoreDirectory* pDirOnStore,
+	BackupStoreDirectory::Entry* pEntry,
+	const std::string &rFilename)
 {
 	// First, delete the directory
 	BackupProtocolClient &connection(rParams.mrContext.GetConnection());
-	connection.QueryDeleteDirectory(ObjectID);
+	connection.QueryDeleteDirectory(pEntry->GetObjectID());
 
+	BackupStoreFilenameClear clear(pEntry->GetName());
+	rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted(
+		pEntry->GetObjectID(), clear.GetClearFilename());
+
 	// Then, delete any directory record
-	std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(rFilename));
+	std::map<std::string, BackupClientDirectoryRecord *>::iterator
+		e(mSubDirectories.find(rFilename));
+
 	if(e != mSubDirectories.end())
 	{
 		// A record exists for this, remove it
@@ -1300,16 +1384,30 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &, const std::string &, const BackupStoreFilename &, int64_t, box_time_t, box_time_t, bool)
-//		Purpose: Private. Upload a file to the server -- may send a patch instead of the whole thing
+//		Name:    BackupClientDirectoryRecord::UploadFile(
+//			 BackupClientDirectoryRecord::SyncParams &,
+//			 const std::string &,
+//			 const BackupStoreFilename &,
+//			 int64_t, box_time_t, box_time_t, bool)
+//		Purpose: Private. Upload a file to the server. May send
+//			 a patch instead of the whole thing
 //		Created: 20/1/04
 //
 // --------------------------------------------------------------------------
-int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
-			int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer)
+int64_t BackupClientDirectoryRecord::UploadFile(
+	BackupClientDirectoryRecord::SyncParams &rParams,
+	const std::string &rFilename,
+	const BackupStoreFilename &rStoreFilename,
+	int64_t FileSize,
+	box_time_t ModificationTime,
+	box_time_t AttributesHash,
+	bool NoPreviousVersionOnServer)
 {
+	BackupClientContext& rContext(rParams.mrContext);
+	ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
 	// Get the connection
-	BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+	BackupProtocolClient &connection(rContext.GetConnection());
 
 	// Info
 	int64_t objID = 0;
@@ -1318,8 +1416,10 @@
 	// Use a try block to catch store full errors
 	try
 	{
-		// Might an old version be on the server, and is the file size over the diffing threshold?
-		if(!NoPreviousVersionOnServer && FileSize >= rParams.mDiffingUploadSizeThreshold)
+		// Might an old version be on the server, and is the file
+		// size over the diffing threshold?
+		if(!NoPreviousVersionOnServer &&
+			FileSize >= rParams.mDiffingUploadSizeThreshold)
 		{
 			// YES -- try to do diff, if possible
 			// First, query the server to see if there's an old version available
@@ -1329,7 +1429,7 @@
 			if(diffFromID != 0)
 			{
 				// Found an old version
-				rParams.GetProgressNotifier().NotifyFileUploadingPatch(this, 
+				rNotifier.NotifyFileUploadingPatch(this, 
 					rFilename);
 
 				// Get the index
@@ -1339,7 +1439,7 @@
 				// Diff the file
 				//
 
-				rParams.mrContext.ManageDiffProcess();
+				rContext.ManageDiffProcess();
 
 				bool isCompletelyDifferent = false;
 				std::auto_ptr<IOStream> patchStream(
@@ -1348,11 +1448,11 @@
 						mObjectID,	/* containing directory */
 						rStoreFilename, diffFromID, *blockIndexStream,
 						connection.GetTimeout(), 
-						&rParams.mrContext, // DiffTimer implementation
+						&rContext, // DiffTimer implementation
 						0 /* not interested in the modification time */, 
 						&isCompletelyDifferent));
 	
-				rParams.mrContext.UnManageDiffProcess();
+				rContext.UnManageDiffProcess();
 
 				//
 				// Upload the patch to the store
@@ -1360,6 +1460,9 @@
 				std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime,
 						AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream));
 				
+				// Get object ID from the result		
+				objID = stored->GetObjectID();
+
 				// Don't attempt to upload it again!
 				doNormalUpload = false;
 			} 
@@ -1368,8 +1471,7 @@
 		if(doNormalUpload)
 		{
 			// below threshold or nothing to diff from, so upload whole
-			rParams.GetProgressNotifier().NotifyFileUploading(this, 
-				rFilename);
+			rNotifier.NotifyFileUploading(this, rFilename);
 			
 			// Prepare to upload, getting a stream which will encode the file as we go along
 			std::auto_ptr<IOStream> upload(
@@ -1390,7 +1492,7 @@
 	}
 	catch(BoxException &e)
 	{
-		rParams.mrContext.UnManageDiffProcess();
+		rContext.UnManageDiffProcess();
 
 		if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply)
 		{
@@ -1404,10 +1506,13 @@
 				{
 					// The hard limit was exceeded on the server, notify!
 					rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull);
+					// return an error code instead of
+					// throwing an exception that we
+					// can't debug.
+					return 0;
 				}
-				rParams.GetProgressNotifier()
-					.NotifyFileUploadServerError(
-						this, rFilename, type, subtype);
+				rNotifier.NotifyFileUploadServerError(this,
+					rFilename, type, subtype);
 			}
 		}
 		
@@ -1415,7 +1520,7 @@
 		throw;
 	}
 
-	rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize);
+	rNotifier.NotifyFileUploaded(this, rFilename, FileSize);
 
 	// Return the new object ID of this file
 	return objID;
@@ -1457,9 +1562,8 @@
 //
 // --------------------------------------------------------------------------
 BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, 
-	ProgressNotifier &rProgressNotifier, BackupClientContext &rContext)
-	: mrProgressNotifier(rProgressNotifier),
-	  mSyncPeriodStart(0),
+	BackupClientContext &rContext)
+	: mSyncPeriodStart(0),
 	  mSyncPeriodEnd(0),
 	  mMaxUploadWait(0),
 	  mMaxFileTimeInFuture(99999999999999999LL),

Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h	2008-05-28 15:24:05 UTC (rev 2181)
@@ -96,6 +96,12 @@
 		const BackupClientDirectoryRecord* pDirRecord,
 		const std::string& rLocalPath,
 		int64_t FileSize) = 0;
+	virtual void NotifyDirectoryDeleted(
+		int64_t ObjectID,
+		const std::string& rRemotePath) = 0;
+	virtual void NotifyFileDeleted(
+		int64_t ObjectID,
+		const std::string& rRemotePath) = 0;
 };
 
 // --------------------------------------------------------------------------
@@ -137,14 +143,12 @@
 	public:
 		SyncParams(
 			BackupDaemon &rDaemon,
-			ProgressNotifier &rProgressNotifier,
 			BackupClientContext &rContext);
 		~SyncParams();
 	private:
 		// No copying
 		SyncParams(const SyncParams&);
 		SyncParams &operator=(const SyncParams&);
-		ProgressNotifier &mrProgressNotifier;
 		
 	public:
 		// Data members are public, as accessors are not justified here
@@ -161,42 +165,52 @@
 		// Member variables modified by syncing process
 		box_time_t mUploadAfterThisTimeInTheFuture;
 		bool mHaveLoggedWarningAboutFutureFileTimes;
-	
-		ProgressNotifier& GetProgressNotifier() const 
-		{ 
-			return mrProgressNotifier;
-		}
 	};
 
-	void SyncDirectory(SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath,
+	void SyncDirectory(SyncParams &rParams,
+		int64_t ContainingDirectoryID,
+		const std::string &rLocalPath,
+		const std::string &rRemotePath,
 		bool ThisDirHasJustBeenCreated = false);
 
 private:
 	void DeleteSubDirectories();
 	BackupStoreDirectory *FetchDirectoryListing(SyncParams &rParams);
-	void UpdateAttributes(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath);
-	bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+	void UpdateAttributes(SyncParams &rParams,
+		BackupStoreDirectory *pDirOnStore,
+		const std::string &rLocalPath);
+	bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath,
+		const std::string &rRemotePath,
+		BackupStoreDirectory *pDirOnStore,
 		std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
-		std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs);
-	int64_t UploadFile(SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
-		int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer);
-	void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, const char *Filename);
-	void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename);
+		std::vector<std::string> &rFiles,
+		const std::vector<std::string> &rDirs);
+	int64_t UploadFile(SyncParams &rParams,
+		const std::string &rFilename,
+		const BackupStoreFilename &rStoreFilename,
+		int64_t FileSize, box_time_t ModificationTime,
+		box_time_t AttributesHash, bool NoPreviousVersionOnServer);
+	void SetErrorWhenReadingFilesystemObject(SyncParams &rParams,
+		const char *Filename);
+	void RemoveDirectoryInPlaceOfFile(SyncParams &rParams,
+		BackupStoreDirectory* pDirOnStore,
+		BackupStoreDirectory::Entry* pEntry,
+		const std::string &rFilename);
 
 private:
-	int64_t 		mObjectID;
+	int64_t 	mObjectID;
 	std::string 	mSubDirName;
-	bool 			mInitialSyncDone;
-	bool 			mSyncDone;
+	bool 		mInitialSyncDone;
+	bool 		mSyncDone;
 
 	// Checksum of directory contents and attributes, used to detect changes
 	uint8_t mStateChecksum[MD5Digest::DigestLength];
 
-	std::map<std::string, box_time_t>						*mpPendingEntries;
-	std::map<std::string, BackupClientDirectoryRecord *>	mSubDirectories;
+	std::map<std::string, box_time_t> *mpPendingEntries;
+	std::map<std::string, BackupClientDirectoryRecord *> mSubDirectories;
 	// mpPendingEntries is a pointer rather than simple a member
-	// variables, because most of the time it'll be empty. This would waste a lot
-	// of memory because of STL allocation policies.
+	// variable, because most of the time it'll be empty. This would
+	// waste a lot of memory because of STL allocation policies.
 };
 
 #endif // BACKUPCLIENTDIRECTORYRECORD__H

Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupDaemon.cpp	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupDaemon.cpp	2008-05-28 15:24:05 UTC (rev 2181)
@@ -47,7 +47,6 @@
 #include "BoxPortsAndFiles.h"
 
 #include "SSLLib.h"
-#include "TLSContext.h"
 
 #include "BackupDaemon.h"
 #include "BackupDaemonConfigVerify.h"
@@ -124,6 +123,10 @@
 	: mState(BackupDaemon::State_Initialising),
 	  mpCommandSocketInfo(0),
 	  mDeleteUnusedRootDirEntriesAfter(0),
+	  mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown),
+	  mStorageLimitExceeded(false),
+	  mReadErrorsOnFilesystemObjects(false),
+	  mLastSyncTime(0),
 	  mLogAllFileAccess(false)
 	#ifdef WIN32
 	, mInstallService(false),
@@ -294,7 +297,7 @@
 	// Clear the contents of the map, so it is empty
 	mLocations.clear();
 	
-	// And delete everything from the assoicated mount vector
+	// And delete everything from the associated mount vector
 	mIDMapMounts.clear();
 }
 
@@ -685,6 +688,20 @@
 	Timers::Cleanup();
 }
 
+void BackupDaemon::InitCrypto()
+{
+	// Read in the certificates creating a TLS context
+	const Configuration &conf(GetConfiguration());
+	std::string certFile(conf.GetKeyValue("CertificateFile"));
+	std::string keyFile(conf.GetKeyValue("PrivateKeyFile"));
+	std::string caFile(conf.GetKeyValue("TrustedCAsFile"));
+	mTlsContext.Initialise(false /* as client */, certFile.c_str(),
+		keyFile.c_str(), caFile.c_str());
+	
+	// Set up the keys for various things
+	BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
+}
+
 // --------------------------------------------------------------------------
 //
 // Function
@@ -695,60 +712,27 @@
 // --------------------------------------------------------------------------
 void BackupDaemon::Run2()
 {
-	// Read in the certificates creating a TLS context
-	TLSContext tlsContext;
+	InitCrypto();
+
 	const Configuration &conf(GetConfiguration());
-	std::string certFile(conf.GetKeyValue("CertificateFile"));
-	std::string keyFile(conf.GetKeyValue("PrivateKeyFile"));
-	std::string caFile(conf.GetKeyValue("TrustedCAsFile"));
-	tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str());
-	
-	// Set up the keys for various things
-	BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
 
-	// Setup various timings
-	int maximumDiffingTime = 600;
-	int keepAliveTime = 60;
-
-	// max diffing time, keep-alive time
-	if(conf.KeyExists("MaximumDiffingTime"))
-	{
-		maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime");
-	}
-	if(conf.KeyExists("KeepAliveTime"))
-	{
-		keepAliveTime = conf.GetKeyValueInt("KeepAliveTime");
-	}
-
 	// How often to connect to the store (approximate)
-	box_time_t updateStoreInterval = SecondsToBoxTime(conf.GetKeyValueInt("UpdateStoreInterval"));
+	box_time_t updateStoreInterval = SecondsToBoxTime(
+		conf.GetKeyValueInt("UpdateStoreInterval"));
 
 	// But are we connecting automatically?
 	bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup");
 	
-	// The minimum age a file needs to be before it will be considered for uploading
-	box_time_t minimumFileAge = SecondsToBoxTime(conf.GetKeyValueInt("MinimumFileAge"));
-
-	// The maximum time we'll wait to upload a file, regardless of how often it's modified
-	box_time_t maxUploadWait = SecondsToBoxTime(conf.GetKeyValueInt("MaxUploadWait"));
-	// Adjust by subtracting the minimum file age, so is relative to sync period end in comparisons
-	maxUploadWait = (maxUploadWait > minimumFileAge)?(maxUploadWait - minimumFileAge):(0);
-
 	// When the next sync should take place -- which is ASAP
 	box_time_t nextSyncTime = 0;
 
 	// When the last sync started (only updated if the store was not full when the sync ended)
-	box_time_t lastSyncTime = 0;
+	mLastSyncTime = 0;
 
  	// --------------------------------------------------------------------------------------------
  
-	// And what's the current client store marker?
-	int64_t clientStoreMarker = 
-		BackupClientContext::ClientStoreMarker_NotKnown;
-	// haven't contacted the store yet
-
  	bool deleteStoreObjectInfoFile = DeserializeStoreObjectInfo(
-		clientStoreMarker, lastSyncTime, nextSyncTime);
+		mLastSyncTime, nextSyncTime);
  
 	// --------------------------------------------------------------------------------------------
 	
@@ -756,6 +740,8 @@
 	// Set state
 	SetState(State_Idle);
 
+	bool doSyncForcedByPreviousSyncError = false;
+
 	// Loop around doing backups
 	do
 	{
@@ -765,70 +751,75 @@
 		bool doSyncForcedByCommand = false;
 
 		// Is a delay necessary?
+		box_time_t currentTime;
+
+		do
 		{
-			box_time_t currentTime;
-			do
+			// Check whether we should be stopping,
+			// and don't run a sync if so.
+			if(StopRun()) break;
+			
+			currentTime = GetCurrentBoxTime();
+
+			// Pause a while, but no more than 
+			// MAX_SLEEP_TIME seconds (use the conditional
+			// because times are unsigned)
+			box_time_t requiredDelay = 
+				(nextSyncTime < currentTime)
+				? (0)
+				: (nextSyncTime - currentTime);
+
+			// If there isn't automatic backup happening, 
+			// set a long delay. And limit delays at the 
+			// same time.
+			if(!automaticBackup && !doSyncForcedByPreviousSyncError)
 			{
-				// Check whether we should be stopping,
-				// and don't run a sync if so.
-				if(StopRun()) break;
-				
-				currentTime = GetCurrentBoxTime();
-	
-				// Pause a while, but no more than 
-				// MAX_SLEEP_TIME seconds (use the conditional
-				// because times are unsigned)
-				box_time_t requiredDelay = 
-					(nextSyncTime < currentTime)
-					? (0)
-					: (nextSyncTime - currentTime);
+				requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME);
+			}
+			else if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME))
+			{
+				requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME);
+			}
 
-				// If there isn't automatic backup happening, 
-				// set a long delay. And limit delays at the 
-				// same time.
-				if(!automaticBackup || requiredDelay > 
-					SecondsToBoxTime(MAX_SLEEP_TIME))
+			// Only delay if necessary
+			if(requiredDelay > 0)
+			{
+				// Sleep somehow. There are choices 
+				// on how this should be done,
+				// depending on the state of the 
+				// control connection
+				if(mpCommandSocketInfo != 0)
 				{
-					requiredDelay = SecondsToBoxTime(
-						MAX_SLEEP_TIME);
+					// A command socket exists, 
+					// so sleep by waiting on it
+					WaitOnCommandSocket(requiredDelay,
+						doSync, doSyncForcedByCommand);
 				}
-
-				// Only delay if necessary
-				if(requiredDelay > 0)
+				else
 				{
-					// Sleep somehow. There are choices 
-					// on how this should be done,
-					// depending on the state of the 
-					// control connection
-					if(mpCommandSocketInfo != 0)
-					{
-						// A command socket exists, 
-						// so sleep by waiting on it
-						WaitOnCommandSocket(
-							requiredDelay, doSync, 
-							doSyncForcedByCommand);
-					}
-					else
-					{
-						// No command socket or 
-						// connection, just do a 
-						// normal sleep
-						time_t sleepSeconds = 
-							BoxTimeToSeconds(
-							requiredDelay);
-						::sleep((sleepSeconds <= 0)
-							? 1
-							: sleepSeconds);
-					}
+					// No command socket or 
+					// connection, just do a 
+					// normal sleep
+					time_t sleepSeconds = 
+						BoxTimeToSeconds(requiredDelay);
+					::sleep((sleepSeconds <= 0)
+						? 1 : sleepSeconds);
 				}
-				
-			} while((!automaticBackup || (currentTime < nextSyncTime)) && !doSync && !StopRun());
+			}
+			
+			if ((automaticBackup || doSyncForcedByPreviousSyncError)
+				&& currentTime >= nextSyncTime)
+			{
+				doSync = true;
+			}
 		}
+		while(!doSync && !StopRun());
 
 		// Time of sync start, and if it's time for another sync 
 		// (and we're doing automatic syncs), set the flag
 		box_time_t currentSyncStartTime = GetCurrentBoxTime();
-		if(automaticBackup && currentSyncStartTime >= nextSyncTime)
+		if((automaticBackup || doSyncForcedByPreviousSyncError) &&
+			currentSyncStartTime >= nextSyncTime)
 		{
 			doSync = true;
 		}
@@ -859,55 +850,6 @@
 			// Reset statistics on uploads
 			BackupStoreFile::ResetStats();
 			
-			// Calculate the sync period of files to examine
-			box_time_t syncPeriodStart = lastSyncTime;
-			box_time_t syncPeriodEnd = currentSyncStartTime - 
-				minimumFileAge;
-
-			if(syncPeriodStart >= syncPeriodEnd &&
-				syncPeriodStart - syncPeriodEnd < minimumFileAge)
-			{
-				// This can happen if we receive a force-sync
-				// command less than minimumFileAge after
-				// the last sync. Deal with it by moving back
-				// syncPeriodStart, which should not do any
-				// damage.
-				syncPeriodStart = syncPeriodEnd -
-					SecondsToBoxTime(1);
-			}
-
-			if(syncPeriodStart >= syncPeriodEnd)
-			{
-				BOX_ERROR("Invalid (negative) sync period: "
-					"perhaps your clock is going "
-					"backwards (" << syncPeriodStart <<
-					" to " << syncPeriodEnd << ")");
-				THROW_EXCEPTION(ClientException,
-					ClockWentBackwards);
-			}
-
-			// Check logic
-			ASSERT(syncPeriodEnd > syncPeriodStart);
-			// Paranoid check on sync times
-			if(syncPeriodStart >= syncPeriodEnd) continue;
-			
-			// Adjust syncPeriodEnd to emulate snapshot 
-			// behaviour properly
-			box_time_t syncPeriodEndExtended = syncPeriodEnd;
-			// Using zero min file age?
-			if(minimumFileAge == 0)
-			{
-				// Add a year on to the end of the end time,
-				// to make sure we sync files which are 
-				// modified after the scan run started.
-				// Of course, they may be eligible to be 
-				// synced again the next time round,
-				// but this should be OK, because the changes 
-				// only upload should upload no data.
-				syncPeriodEndExtended += SecondsToBoxTime(
-					(time_t)(356*24*3600));
-			}
-
 			// Delete the serialised store object file,
 			// so that we don't try to reload it after a
 			// partially completed backup
@@ -933,115 +875,13 @@
 
 			try
 			{
-				// Set state and log start
-				SetState(State_Connected);
-				BOX_NOTICE("Beginning scan of local files");
-
-				std::string extendedLogFile;
-				if (conf.KeyExists("ExtendedLogFile"))
-				{
-					extendedLogFile = conf.GetKeyValue(
-						"ExtendedLogFile");
-				}
-				
-				if (conf.KeyExists("LogAllFileAccess"))
-				{
-					mLogAllFileAccess = 
-						conf.GetKeyValueBool(
-							"LogAllFileAccess");
-				}
-				
-				// Then create a client context object (don't 
-				// just connect, as this may be unnecessary)
-				BackupClientContext clientContext
-				(
-					*this, 
-					tlsContext, 
-					conf.GetKeyValue("StoreHostname"),
-					conf.GetKeyValueInt("StorePort"),
-					conf.GetKeyValueInt("AccountNumber"), 
-					conf.GetKeyValueBool("ExtendedLogging"),
-					conf.KeyExists("ExtendedLogFile"),
-					extendedLogFile
-				);
-					
-				// Set up the sync parameters
-				BackupClientDirectoryRecord::SyncParams params(
-					*this, *this, clientContext);
-				params.mSyncPeriodStart = syncPeriodStart;
-				params.mSyncPeriodEnd = syncPeriodEndExtended;
-				// use potentially extended end time
-				params.mMaxUploadWait = maxUploadWait;
-				params.mFileTrackingSizeThreshold = 
-					conf.GetKeyValueInt(
-					"FileTrackingSizeThreshold");
-				params.mDiffingUploadSizeThreshold = 
-					conf.GetKeyValueInt(
-					"DiffingUploadSizeThreshold");
-				params.mMaxFileTimeInFuture = 
-					SecondsToBoxTime(
-						conf.GetKeyValueInt(
-							"MaxFileTimeInFuture"));
-				mDeleteRedundantLocationsAfter =
-					conf.GetKeyValueInt(
-						"DeleteRedundantLocationsAfter");
-
-				clientContext.SetMaximumDiffingTime(maximumDiffingTime);
-				clientContext.SetKeepAliveTime(keepAliveTime);
-				
-				// Set store marker
-				clientContext.SetClientStoreMarker(clientStoreMarker);
-				
-				// Set up the locations, if necessary -- 
-				// need to do it here so we have a 
-				// (potential) connection to use
-				if(mLocations.empty())
-				{
-					const Configuration &locations(
-						conf.GetSubConfiguration(
-							"BackupLocations"));
-					
-					// Make sure all the directory records
-					// are set up
-					SetupLocations(clientContext, locations);
-				}
-				
-				// Get some ID maps going
-				SetupIDMapsForSync();
-
-				// Delete any unused directories?
-				DeleteUnusedRootDirEntries(clientContext);
-								
 				// Notify administrator
 				NotifySysadmin(NotifyEvent_BackupStart);
 
-				// Go through the records, syncing them
-				for(std::vector<Location *>::const_iterator 
-					i(mLocations.begin()); 
-					i != mLocations.end(); ++i)
-				{
-					// Set current and new ID map pointers
-					// in the context
-					clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]);
-				
-					// Set exclude lists (context doesn't
-					// take ownership)
-					clientContext.SetExcludeLists(
-						(*i)->mpExcludeFiles,
-						(*i)->mpExcludeDirs);
+				RunSyncNow();
 
-					// Sync the directory
-					(*i)->mpDirectoryRecord->SyncDirectory(
-						params,
-						BackupProtocolClientListDirectory::RootDirectory,
-						(*i)->mPath);
-
-					// Unset exclude lists (just in case)
-					clientContext.SetExcludeLists(0, 0);
-				}
-				
 				// Errors reading any files?
-				if(params.mReadErrorsOnFilesystemObjects)
+				if(mReadErrorsOnFilesystemObjects)
 				{
 					// Notify administrator
 					NotifySysadmin(NotifyEvent_ReadError);
@@ -1053,33 +893,14 @@
 					mNotificationsSent[NotifyEvent_ReadError] = false;
 				}
 				
-				// Perform any deletions required -- these are
-				// delayed until the end to allow renaming to 
-				// happen neatly.
-				clientContext.PerformDeletions();
-
-				// Close any open connection
-				clientContext.CloseAnyOpenConnection();
-				
-				// Get the new store marker
-				clientStoreMarker = clientContext.GetClientStoreMarker();
-				
 				// Check the storage limit
-				if(clientContext.StorageLimitExceeded())
+				if(mStorageLimitExceeded)
 				{
 					// Tell the sysadmin about this
 					NotifySysadmin(NotifyEvent_StoreFull);
 				}
 				else
 				{
-					// The start time of the next run is
-					// the end time of this run.
-					// This is only done if the storage
-					// limit wasn't exceeded (as things
-					// won't have been done properly if
-					// it was)
-					lastSyncTime = syncPeriodEnd;
-
 					// unflag the storage full notify flag
 					// so that next time the store is full,
 					// an alert will be sent
@@ -1092,12 +913,6 @@
 					Random::RandomInt(updateStoreInterval >>
 					SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
 			
-				// Commit the ID Maps
-				CommitIDMapsAfterSync();
-
-				// Log
-				BOX_NOTICE("Finished scan of local files");
-
 				// Notify administrator
 				NotifySysadmin(NotifyEvent_BackupFinish);
 
@@ -1108,11 +923,14 @@
 				// delete the file next time we start a backup
 
 				deleteStoreObjectInfoFile = 
-					SerializeStoreObjectInfo(
-						clientStoreMarker, 
-						lastSyncTime, nextSyncTime);
+					SerializeStoreObjectInfo(mLastSyncTime,
+						nextSyncTime);
 
 				// --------------------------------------------------------------------------------------------
+
+				// If we were retrying after an error,
+				// now would be a good time to stop :-)
+				doSyncForcedByPreviousSyncError = false;
 			}
 			catch(BoxException &e)
 			{
@@ -1153,9 +971,9 @@
 				}
 
 				// Clear state data
-				syncPeriodStart = 0;
-				// go back to beginning of time
-				clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;	// no store marker, so download everything
+				// Go back to beginning of time
+				mLastSyncTime = 0;
+				mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;	// no store marker, so download everything
 				DeleteAllLocations();
 				DeleteAllIDMaps();
 
@@ -1168,7 +986,8 @@
 					return;
 				}
 
-				// If the Berkely db files get corrupted, delete them and try again immediately
+				// If the Berkely db files get corrupted,
+				// delete them and try again immediately.
 				if(isBerkelyDbFailure)
 				{
 					BOX_ERROR("Berkely db inode map files corrupted, deleting and restarting scan. Renamed files and directories will not be tracked until after this scan.");
@@ -1188,10 +1007,11 @@
 						"waiting to retry...");
 					::sleep(10);
 					nextSyncTime = currentSyncStartTime + 
-						SecondsToBoxTime(90) +
+						SecondsToBoxTime(100) +
 						Random::RandomInt(
 							updateStoreInterval >> 
 							SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
+					doSyncForcedByPreviousSyncError = true;
 				}
 			}
 
@@ -1221,7 +1041,240 @@
 	DeleteAllIDMaps();
 }
 
+void BackupDaemon::RunSyncNow()
+{
+	// Set state and log start
+	SetState(State_Connected);
+	BOX_NOTICE("Beginning scan of local files");
 
+	const Configuration &conf(GetConfiguration());
+
+	std::string extendedLogFile;
+	if (conf.KeyExists("ExtendedLogFile"))
+	{
+		extendedLogFile = conf.GetKeyValue("ExtendedLogFile");
+	}
+	
+	if (conf.KeyExists("LogAllFileAccess"))
+	{
+		mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess");
+	}
+	
+	// Then create a client context object (don't 
+	// just connect, as this may be unnecessary)
+	BackupClientContext clientContext
+	(
+		*this, 
+		mTlsContext, 
+		conf.GetKeyValue("StoreHostname"),
+		conf.GetKeyValueInt("StorePort"),
+		conf.GetKeyValueInt("AccountNumber"), 
+		conf.GetKeyValueBool("ExtendedLogging"),
+		conf.KeyExists("ExtendedLogFile"),
+		extendedLogFile, *this
+	);
+		
+	// The minimum age a file needs to be before it will be
+	// considered for uploading
+	box_time_t minimumFileAge = SecondsToBoxTime(
+		conf.GetKeyValueInt("MinimumFileAge"));
+
+	// The maximum time we'll wait to upload a file, regardless
+	// of how often it's modified
+	box_time_t maxUploadWait = SecondsToBoxTime(
+		conf.GetKeyValueInt("MaxUploadWait"));
+	// Adjust by subtracting the minimum file age, so is relative
+	// to sync period end in comparisons
+	if (maxUploadWait > minimumFileAge)
+	{
+		maxUploadWait -= minimumFileAge;
+	}
+	else
+	{
+		maxUploadWait = 0;
+	}
+
+	// Calculate the sync period of files to examine
+	box_time_t syncPeriodStart = mLastSyncTime;
+	box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge;
+
+	if(syncPeriodStart >= syncPeriodEnd &&
+		syncPeriodStart - syncPeriodEnd < minimumFileAge)
+	{
+		// This can happen if we receive a force-sync
+		// command less than minimumFileAge after
+		// the last sync. Deal with it by moving back
+		// syncPeriodStart, which should not do any
+		// damage.
+		syncPeriodStart = syncPeriodEnd -
+			SecondsToBoxTime(1);
+	}
+
+	if(syncPeriodStart >= syncPeriodEnd)
+	{
+		BOX_ERROR("Invalid (negative) sync period: "
+			"perhaps your clock is going "
+			"backwards (" << syncPeriodStart <<
+			" to " << syncPeriodEnd << ")");
+		THROW_EXCEPTION(ClientException,
+			ClockWentBackwards);
+	}
+
+	// Check logic
+	ASSERT(syncPeriodEnd > syncPeriodStart);
+	// Paranoid check on sync times
+	if(syncPeriodStart >= syncPeriodEnd) return;
+	
+	// Adjust syncPeriodEnd to emulate snapshot 
+	// behaviour properly
+	box_time_t syncPeriodEndExtended = syncPeriodEnd;
+
+	// Using zero min file age?
+	if(minimumFileAge == 0)
+	{
+		// Add a year on to the end of the end time,
+		// to make sure we sync files which are 
+		// modified after the scan run started.
+		// Of course, they may be eligible to be 
+		// synced again the next time round,
+		// but this should be OK, because the changes 
+		// only upload should upload no data.
+		syncPeriodEndExtended += SecondsToBoxTime(
+			(time_t)(356*24*3600));
+	}
+
+	// Set up the sync parameters
+	BackupClientDirectoryRecord::SyncParams params(
+		*this, clientContext);
+	params.mSyncPeriodStart = syncPeriodStart;
+	params.mSyncPeriodEnd = syncPeriodEndExtended;
+	// use potentially extended end time
+	params.mMaxUploadWait = maxUploadWait;
+	params.mFileTrackingSizeThreshold = 
+		conf.GetKeyValueInt(
+		"FileTrackingSizeThreshold");
+	params.mDiffingUploadSizeThreshold = 
+		conf.GetKeyValueInt(
+		"DiffingUploadSizeThreshold");
+	params.mMaxFileTimeInFuture = 
+		SecondsToBoxTime(
+			conf.GetKeyValueInt(
+				"MaxFileTimeInFuture"));
+	mDeleteRedundantLocationsAfter =
+		conf.GetKeyValueInt(
+			"DeleteRedundantLocationsAfter");
+	mStorageLimitExceeded = false;
+	mReadErrorsOnFilesystemObjects = false;
+
+	// Setup various timings
+	int maximumDiffingTime = 600;
+	int keepAliveTime = 60;
+
+	// max diffing time, keep-alive time
+	if(conf.KeyExists("MaximumDiffingTime"))
+	{
+		maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime");
+	}
+	if(conf.KeyExists("KeepAliveTime"))
+	{
+		keepAliveTime = conf.GetKeyValueInt("KeepAliveTime");
+	}
+
+	clientContext.SetMaximumDiffingTime(maximumDiffingTime);
+	clientContext.SetKeepAliveTime(keepAliveTime);
+	
+	// Set store marker
+	clientContext.SetClientStoreMarker(mClientStoreMarker);
+	
+	// Set up the locations, if necessary -- 
+	// need to do it here so we have a 
+	// (potential) connection to use
+	if(mLocations.empty())
+	{
+		const Configuration &locations(
+			conf.GetSubConfiguration(
+				"BackupLocations"));
+		
+		// Make sure all the directory records
+		// are set up
+		SetupLocations(clientContext, locations);
+	}
+	
+	// Get some ID maps going
+	SetupIDMapsForSync();
+
+	// Delete any unused directories?
+	DeleteUnusedRootDirEntries(clientContext);
+					
+	// Go through the records, syncing them
+	for(std::vector<Location *>::const_iterator 
+		i(mLocations.begin()); 
+		i != mLocations.end(); ++i)
+	{
+		// Set current and new ID map pointers
+		// in the context
+		clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex],
+			mNewIDMaps[(*i)->mIDMapIndex]);
+	
+		// Set exclude lists (context doesn't
+		// take ownership)
+		clientContext.SetExcludeLists(
+			(*i)->mpExcludeFiles,
+			(*i)->mpExcludeDirs);
+
+		// Sync the directory
+		(*i)->mpDirectoryRecord->SyncDirectory(
+			params,
+			BackupProtocolClientListDirectory::RootDirectory,
+			(*i)->mPath, std::string("/") + (*i)->mName);
+
+		// Unset exclude lists (just in case)
+		clientContext.SetExcludeLists(0, 0);
+	}
+	
+	// Errors reading any files?
+	if(params.mReadErrorsOnFilesystemObjects)
+	{
+		// Notify administrator
+		NotifySysadmin(NotifyEvent_ReadError);
+	}
+	else
+	{
+		// Unset the read error flag, so the						// error is reported again if it
+		// happens again
+		mNotificationsSent[NotifyEvent_ReadError] = false;
+	}
+	
+	// Perform any deletions required -- these are
+	// delayed until the end to allow renaming to 
+	// happen neatly.
+	clientContext.PerformDeletions();
+
+	// Close any open connection
+	clientContext.CloseAnyOpenConnection();
+	
+	// Get the new store marker
+	mClientStoreMarker = clientContext.GetClientStoreMarker();
+	mStorageLimitExceeded = clientContext.StorageLimitExceeded();
+	mReadErrorsOnFilesystemObjects =
+		params.mReadErrorsOnFilesystemObjects;
+
+	if(!mStorageLimitExceeded)
+	{
+		// The start time of the next run is the end time of this
+		// run. This is only done if the storage limit wasn't
+		// exceeded (as things won't have been done properly if
+		// it was)
+		mLastSyncTime = syncPeriodEnd;
+	}
+
+	// Commit the ID Maps
+	CommitIDMapsAfterSync();
+
+	// Log
+	BOX_NOTICE("Finished scan of local files");
+}
+
 // --------------------------------------------------------------------------
 //
 // Function
@@ -2460,13 +2513,13 @@
 	// Entries to delete, and it's the right time to do so...
 	BOX_NOTICE("Deleting unused locations from store root...");
 	BackupProtocolClient &connection(rContext.GetConnection());
-	for(std::vector<std::pair<int64_t,std::string> >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i)
+	for(std::vector<std::pair<int64_t,std::string> >::iterator
+		i(mUnusedRootDirEntries.begin());
+		i != mUnusedRootDirEntries.end(); ++i)
 	{
 		connection.QueryDeleteDirectory(i->first);
-		
-		// Log this
-		BOX_NOTICE("Deleted " << i->second << " (ID " << i->first
-			<< ") from store root");
+		rContext.GetProgressNotifier().NotifyFileDeleted(
+			i->first, i->second);
 	}
 
 	// Reset state
@@ -2737,9 +2790,12 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime)
-//		Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction.
-//
+//		Name:    BackupDaemon::SerializeStoreObjectInfo(
+//			 box_time_t theLastSyncTime,
+//			 box_time_t theNextSyncTime)
+//		Purpose: Serializes remote directory and file information
+//			 into a stream of bytes, using an Archive
+//			 abstraction.
 //		Created: 2005/04/11
 //
 // --------------------------------------------------------------------------
@@ -2748,7 +2804,8 @@
 static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE";
 static const int STOREOBJECTINFO_VERSION = 2;
 
-bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const
+bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime,
+	box_time_t theNextSyncTime) const
 {
 	if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
 	{
@@ -2777,7 +2834,7 @@
 		anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); 
 		anArchive.Write(STOREOBJECTINFO_VERSION);
 		anArchive.Write(GetLoadedConfigModifiedTime());
-		anArchive.Write(aClientStoreMarker);
+		anArchive.Write(mClientStoreMarker);
 		anArchive.Write(theLastSyncTime);
 		anArchive.Write(theNextSyncTime);
 
@@ -2829,15 +2886,13 @@
 	}
 	catch(std::exception &e)
 	{
-		BOX_ERROR("Internal error writing store object "
-			"info file (" << StoreObjectInfoFile << "): "
-			<< e.what());
+		BOX_ERROR("Failed to write StoreObjectInfoFile: " <<
+			StoreObjectInfoFile << ": " << e.what());
 	}
 	catch(...)
 	{
-		BOX_ERROR("Internal error writing store object "
-			"info file (" << StoreObjectInfoFile << "): "
-			"unknown error");
+		BOX_ERROR("Failed to write StoreObjectInfoFile: " <<
+			StoreObjectInfoFile << ": unknown error");
 	}
 
 	return created;
@@ -2846,13 +2901,17 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime)
-//		Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction.
-//
+//		Name:    BackupDaemon::DeserializeStoreObjectInfo(
+//			 box_time_t & theLastSyncTime,
+//			 box_time_t & theNextSyncTime)
+//		Purpose: Deserializes remote directory and file information
+//			 from a stream of bytes, using an Archive
+//			 abstraction.
 //		Created: 2005/04/11
 //
 // --------------------------------------------------------------------------
-bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime)
+bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime,
+	box_time_t & theNextSyncTime)
 {
 	//
 	//
@@ -2944,7 +3003,7 @@
 		//
 		// this is it, go at it
 		//
-		anArchive.Read(aClientStoreMarker);
+		anArchive.Read(mClientStoreMarker);
 		anArchive.Read(theLastSyncTime);
 		anArchive.Read(theNextSyncTime);
 
@@ -3022,7 +3081,7 @@
 
 	DeleteAllLocations();
 
-	aClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;
+	mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;
 	theLastSyncTime = 0;
 	theNextSyncTime = 0;
 

Modified: box/trunk/bin/bbackupd/BackupDaemon.h
===================================================================
--- box/trunk/bin/bbackupd/BackupDaemon.h	2008-05-28 14:40:39 UTC (rev 2180)
+++ box/trunk/bin/bbackupd/BackupDaemon.h	2008-05-28 15:24:05 UTC (rev 2181)
@@ -14,13 +14,15 @@
 #include <string>
 #include <memory>
 
+#include "BackupClientDirectoryRecord.h"
 #include "BoxTime.h"
 #include "Daemon.h"
-#include "BackupClientDirectoryRecord.h"
+#include "Logging.h"
 #include "Socket.h"
 #include "SocketListen.h"
 #include "SocketStream.h"
-#include "Logging.h"
+#include "TLSContext.h"
+
 #include "autogen_BackupProtocolClient.h"
 
 #ifdef WIN32
@@ -52,10 +54,10 @@
 private:
 	// methods below do partial (specialized) serialization of 
 	// client state only
-	bool SerializeStoreObjectInfo(int64_t aClientStoreMarker, 
-		box_time_t theLastSyncTime, box_time_t theNextSyncTime) const;
-	bool DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, 
-		box_time_t & theLastSyncTime, box_time_t & theNextSyncTime);
+	bool SerializeStoreObjectInfo(box_time_t theLastSyncTime,
+		box_time_t theNextSyncTime) const;
+	bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime,
+		box_time_t & theNextSyncTime);
 	bool DeleteStoreObjectInfo() const;
 	BackupDaemon(const BackupDaemon &);
 
@@ -111,6 +113,11 @@
 private:
 	void Run2();
 
+public:
+	void InitCrypto();
+	void RunSyncNow();
+
+private:
 	void DeleteAllLocations();
 	void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf);
 
@@ -205,8 +212,15 @@
 	box_time_t mDeleteUnusedRootDirEntriesAfter;	// time to delete them
 	std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries;
 
+	int64_t mClientStoreMarker;
+	bool mStorageLimitExceeded;
+	bool mReadErrorsOnFilesystemObjects;
+	box_time_t mLastSyncTime;
+	TLSContext mTlsContext;
+
 public:
  	bool StopRun() { return this->Daemon::StopRun(); }
+	bool StorageLimitExceeded() { return mStorageLimitExceeded; }
  
 private:
 	bool mLogAllFileAccess;
@@ -427,6 +441,28 @@
 			BOX_INFO("Synchronised file: " << rLocalPath);
 		} 
 	}
+	virtual void NotifyDirectoryDeleted(
+		int64_t ObjectID,
+		const std::string& rRemotePath)
+	{
+		if (mLogAllFileAccess)
+		{
+			BOX_NOTICE("Deleted directory: " << rRemotePath << 
+				" (ID " << BOX_FORMAT_OBJECTID(ObjectID) <<
+				")");
+		}
+	}
+	virtual void NotifyFileDeleted(
+		int64_t ObjectID,
+		const std::string& rRemotePath)
+	{
+		if (mLogAllFileAccess)
+		{
+			BOX_NOTICE("Deleted file: " << rRemotePath << 
+				" (ID " << BOX_FORMAT_OBJECTID(ObjectID) <<
+				")");
+		}
+	}
 
 #ifdef WIN32
 	public:



From boxbackup-dev at fluffy.co.uk  Wed May 28 16:25:59 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 16:25:59 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2182 - box/trunk/bin/bbackupquery
Message-ID: <20080528152559.41D46325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 16:25:59 +0100 (Wed, 28 May 2008)
New Revision: 2182

Modified:
   box/trunk/bin/bbackupquery/BackupQueries.cpp
   box/trunk/bin/bbackupquery/BackupQueries.h
Log:
Use symbolic names instead of #defines for return codes.


Modified: box/trunk/bin/bbackupquery/BackupQueries.cpp
===================================================================
--- box/trunk/bin/bbackupquery/BackupQueries.cpp	2008-05-28 15:24:05 UTC (rev 2181)
+++ box/trunk/bin/bbackupquery/BackupQueries.cpp	2008-05-28 15:25:59 UTC (rev 2182)
@@ -122,7 +122,7 @@
 		{
 			BOX_WARNING("System command returned error code " <<
 				result);
-			SetReturnCode(COMMAND_RETURN_ERROR);
+			SetReturnCode(ReturnCode::Command_Error);
 		}
 		return;
 	}
@@ -295,7 +295,7 @@
 	if(cmd != COMMAND_Quit && cmd != COMMAND_Exit)
 	{
 		// If not a quit command, set the return code to zero
-		SetReturnCode(0);
+		SetReturnCode(ReturnCode::Command_OK);
 	}
 
 	// Handle command
@@ -410,7 +410,7 @@
 		{
 			BOX_ERROR("Directory '" << args[0] << "' not found "
 				"on store.");
-			SetReturnCode(COMMAND_RETURN_ERROR);
+			SetReturnCode(ReturnCode::Command_Error);
 			return;
 		}
 	}
@@ -448,13 +448,13 @@
 	catch (std::exception &e)
 	{
 		BOX_ERROR("Failed to list directory: " << e.what());
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	catch (...)
 	{
 		BOX_ERROR("Failed to list directory: unknown error");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 
@@ -773,7 +773,7 @@
 	if(args.size() != 1 || args[0].size() == 0)
 	{
 		BOX_ERROR("Incorrect usage. cd [-o] [-d] <directory>");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 
@@ -791,7 +791,7 @@
 	if(id == 0)
 	{
 		BOX_ERROR("Directory '" << args[0] << "' not found.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	
@@ -813,7 +813,7 @@
 	if(args.size() != 1 || args[0].size() == 0)
 	{
 		BOX_ERROR("Incorrect usage. lcd <local-directory>");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	
@@ -823,7 +823,7 @@
 	if(!ConvertConsoleToUtf8(args[0].c_str(), dirName))
 	{
 		BOX_ERROR("Failed to convert path from console encoding.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	int result = ::chdir(dirName.c_str());
@@ -842,7 +842,7 @@
 				"'" << args[0] << "'");
 		}
 
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	
@@ -851,7 +851,7 @@
 	if(::getcwd(wd, PATH_MAX) == 0)
 	{
 		BOX_LOG_SYS_ERROR("Error getting current directory");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 
@@ -859,7 +859,7 @@
 	if(!ConvertUtf8ToConsole(wd, dirName))
 	{
 		BOX_ERROR("Failed to convert new path from console encoding.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	BOX_INFO("Local current directory is now '" << dirName << "'.");
@@ -1069,7 +1069,7 @@
 	{
 		BOX_ERROR("The local file " << localName << " already exists, "
 			"will not overwrite it.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	
@@ -1268,15 +1268,15 @@
 	{
 		if (params.mUncheckedFiles != 0)
 		{
-			SetReturnCode(COMPARE_RETURN_ERROR);
+			SetReturnCode(ReturnCode::Compare_Error);
 		} 
 		else if (params.mDifferences != 0)
 		{
-			SetReturnCode(COMPARE_RETURN_DIFFERENT);
+			SetReturnCode(ReturnCode::Compare_Different);
 		}
 		else
 		{
-			SetReturnCode(COMPARE_RETURN_SAME);
+			SetReturnCode(ReturnCode::Compare_Same);
 		}
 	}
 }
@@ -2055,13 +2055,13 @@
 	catch(std::exception &e)
 	{
 		BOX_ERROR("Failed to restore: " << e.what());
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 	catch(...)
 	{
 		BOX_ERROR("Failed to restore: unknown exception");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		return;
 	}
 
@@ -2079,30 +2079,30 @@
 	case Restore_ResumePossible:
 		BOX_ERROR("Resume possible -- repeat command with -r flag "
 			"to resume.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		break;
 	
 	case Restore_TargetExists:
 		BOX_ERROR("The target directory exists. You cannot restore "
 			"over an existing directory.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		break;
 		
 	case Restore_TargetPathNotFound:
 		BOX_ERROR("The target directory path does not exist.\n"
 			"To restore to a directory whose parent "
 			"does not exist, create the parent first.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		break;
 
 	case Restore_UnknownError:
 		BOX_ERROR("Unknown error during restore.");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		break;
 
 	default:
 		BOX_ERROR("Unknown restore result " << result << ".");
-		SetReturnCode(COMMAND_RETURN_ERROR);
+		SetReturnCode(ReturnCode::Command_Error);
 		break;
 	}
 }

Modified: box/trunk/bin/bbackupquery/BackupQueries.h
===================================================================
--- box/trunk/bin/bbackupquery/BackupQueries.h	2008-05-28 15:24:05 UTC (rev 2181)
+++ box/trunk/bin/bbackupquery/BackupQueries.h	2008-05-28 15:25:59 UTC (rev 2182)
@@ -79,10 +79,29 @@
 		const ExcludeList *mpExcludeDirs;
 		box_time_t mLatestFileUploadTime;
 	};
-	void CompareLocation(const std::string &rLocation, CompareParams &rParams);
-	void Compare(const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams);
-	void Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams);
+	void CompareLocation(const std::string &rLocation,
+		CompareParams &rParams);
+	void Compare(const std::string &rStoreDir,
+		const std::string &rLocalDir, CompareParams &rParams);
+	void Compare(int64_t DirID, const std::string &rStoreDir,
+		const std::string &rLocalDir, CompareParams &rParams);
 
+public:
+
+	class ReturnCode
+	{
+		public:
+		enum {
+			Command_OK = 0,
+			Compare_Same = 1,
+			Compare_Different,
+			Compare_Error,
+			Command_Error,
+		} Type;
+	};
+
+private:
+
 	// Utility functions
 	int64_t FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion = false,
 		bool AllowDeletedDirs = false, std::vector<std::pair<std::string, int64_t> > *pStack = 0);



From boxbackup-dev at fluffy.co.uk  Wed May 28 16:28:36 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 16:28:36 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2183 - box/trunk/lib/intercept
Message-ID: <20080528152836.5D819325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 16:28:36 +0100 (Wed, 28 May 2008)
New Revision: 2183

Modified:
   box/trunk/lib/intercept/intercept.cpp
   box/trunk/lib/intercept/intercept.h
Log:
Add stat() intercept and stat() and lstat() post call hooks.


Modified: box/trunk/lib/intercept/intercept.cpp
===================================================================
--- box/trunk/lib/intercept/intercept.cpp	2008-05-28 15:25:59 UTC (rev 2182)
+++ box/trunk/lib/intercept/intercept.cpp	2008-05-28 15:28:36 UTC (rev 2183)
@@ -84,7 +84,13 @@
 static lstat_t*    lstat_real    = NULL;
 static lstat_t*    lstat_hook    = NULL;
 static const char* lstat_file    = NULL;
+static lstat_t*    stat_real     = NULL;
+static lstat_t*    stat_hook     = NULL;
+static const char* stat_file     = NULL;
 
+static lstat_post_hook_t* lstat_post_hook = NULL;
+static lstat_post_hook_t* stat_post_hook  = NULL;
+
 #define SIZE_ALWAYS_ERROR	-773
 
 void intercept_clear_setup()
@@ -97,7 +103,10 @@
 	intercept_filepos = 0;
 	intercept_delay_ms = 0;
 	readdir_hook = NULL;
+	stat_hook = NULL;
 	lstat_hook = NULL;
+	stat_post_hook = NULL;
+	lstat_post_hook = NULL;
 }
 
 bool intercept_triggered()
@@ -405,6 +414,40 @@
 	lstat_hook = hookfn;
 }
 
+void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn)
+{
+	/*
+	if (hookfn != NULL)
+	{
+		BOX_TRACE("lstat hooked to " << hookfn << " for " << filename);
+	}
+	else
+	{
+		BOX_TRACE("lstat unhooked from " << lstat_hook << " for " <<
+			lstat_file);
+	}
+	*/
+
+	lstat_post_hook = hookfn;
+}
+
+void intercept_setup_stat_post_hook(lstat_post_hook_t hookfn)
+{
+	/*
+	if (hookfn != NULL)
+	{
+		BOX_TRACE("lstat hooked to " << hookfn << " for " << filename);
+	}
+	else
+	{
+		BOX_TRACE("lstat unhooked from " << lstat_hook << " for " <<
+			lstat_file);
+	}
+	*/
+
+	stat_post_hook = hookfn;
+}
+
 static void * find_function(const char *pName)
 {
 	dlerror();
@@ -534,10 +577,15 @@
 	if (lstat_hook == NULL || strcmp(file_name, lstat_file) != 0)
 	{
 	#ifdef LINUX_WEIRD_LSTAT
-		return lstat_real(ver, file_name, buf);
+		int ret = lstat_real(ver, file_name, buf);
 	#else
-		return lstat_real(file_name, buf);
+		int ret = lstat_real(file_name, buf);
 	#endif
+		if (lstat_post_hook != NULL)
+		{
+			ret = lstat_post_hook(ret, file_name, buf);
+		}
+		return ret;
 	}
 
 	#ifdef LINUX_WEIRD_LSTAT
@@ -547,4 +595,48 @@
 	#endif
 }
 
+extern "C" int 
+#ifdef LINUX_WEIRD_LSTAT
+__xstat(int ver, const char *file_name, STAT_STRUCT *buf)
+#else
+stat(const char *file_name, STAT_STRUCT *buf)
+#endif
+{
+	if (stat_real == NULL)
+	{
+	#ifdef LINUX_WEIRD_LSTAT
+		stat_real = (lstat_t*)find_function("__xstat");
+	#else
+		stat_real = (lstat_t*)find_function("stat");
+	#endif
+	}
+
+	if (stat_real == NULL)
+	{
+		perror("cannot find real stat");
+		errno = ENOSYS;
+		return -1;
+	}
+
+	if (stat_hook == NULL || strcmp(file_name, stat_file) != 0)
+	{
+	#ifdef LINUX_WEIRD_LSTAT
+		int ret = stat_real(ver, file_name, buf);
+	#else
+		int ret = stat_real(file_name, buf);
+	#endif
+		if (stat_post_hook != NULL)
+		{
+			ret = stat_post_hook(ret, file_name, buf);
+		}
+		return ret;
+	}
+
+	#ifdef LINUX_WEIRD_LSTAT
+	return stat_hook(ver, file_name, buf);
+	#else
+	return stat_hook(file_name, buf);
+	#endif
+}
+
 #endif // n PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE

Modified: box/trunk/lib/intercept/intercept.h
===================================================================
--- box/trunk/lib/intercept/intercept.h	2008-05-28 15:25:59 UTC (rev 2182)
+++ box/trunk/lib/intercept/intercept.h	2008-05-28 15:28:36 UTC (rev 2183)
@@ -34,6 +34,9 @@
 #endif
 }
 
+typedef int (lstat_post_hook_t) (int old_ret, const char *file_name,
+	struct stat *buf);
+
 void intercept_setup_error(const char *filename, unsigned int errorafter, 
 	int errortoreturn, int syscalltoerror);
 void intercept_setup_delay(const char *filename, unsigned int delay_after,
@@ -42,6 +45,10 @@
 
 void intercept_setup_readdir_hook(const char *dirname,  readdir_t hookfn);
 void intercept_setup_lstat_hook  (const char *filename, lstat_t   hookfn);
+void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn);
+void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn);
 
+void intercept_clear_setup();
+
 #endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
 #endif // !INTERCEPT_H



From boxbackup-dev at fluffy.co.uk  Wed May 28 16:35:42 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 16:35:42 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2184 - in box/trunk: bin/bbackupquery test/backupstorefix test/backupstorefix/testfiles test/bbackupd test/bbackupd/testfiles
Message-ID: <20080528153542.AC09A325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 16:35:42 +0100 (Wed, 28 May 2008)
New Revision: 2184

Modified:
   box/trunk/bin/bbackupquery/bbackupquery.cpp
   box/trunk/test/backupstorefix/testbackupstorefix.cpp
   box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in
   box/trunk/test/bbackupd/testbbackupd.cpp
   box/trunk/test/bbackupd/testfiles/extcheck1.pl.in
   box/trunk/test/bbackupd/testfiles/extcheck2.pl.in
Log:
Add bbackupquery -W<level> option to set explicit warning level, 

Obsolete old (inconsistent) meaning of -q in bbackupquery.

Replace -q with -Wwarning or -Werror in tests to reduce noise and fix 
tests.

Test that reading a nonexistent directory on the server doesn't crash 
server or client.

Test that bbackupd does continue backup run and delete files when 
storage limit is exceeded.

Use logging guards to hide expected warnings in testbbackupd.

Remove apparently pointless listing files on server at the end of 
testbbackupd.


Modified: box/trunk/bin/bbackupquery/bbackupquery.cpp
===================================================================
--- box/trunk/bin/bbackupquery/bbackupquery.cpp	2008-05-28 15:28:36 UTC (rev 2183)
+++ box/trunk/bin/bbackupquery/bbackupquery.cpp	2008-05-28 15:35:42 UTC (rev 2184)
@@ -107,7 +107,6 @@
 	#endif
 	
 	// Flags
-	bool quiet = false;
 	bool readWrite = false;
 
 	Logging::SetProgramName("Box Backup (bbackupquery)");
@@ -119,10 +118,10 @@
 	#endif
 
 #ifdef WIN32
-	const char* validOpts = "qvwuc:l:";
+	const char* validOpts = "qvwuc:l:W:";
 	bool unicodeConsole = false;
 #else
-	const char* validOpts = "qvwc:l:";
+	const char* validOpts = "qvwc:l:W:";
 #endif
 
 	// See if there's another entry on the command line
@@ -133,9 +132,6 @@
 		{
 			case 'q':
 			{
-				// Quiet mode
-				quiet = true;
-
 				if(masterLevel == Log::NOTHING)
 				{
 					BOX_FATAL("Too many '-q': "
@@ -160,6 +156,17 @@
 			}
 			break;
 
+		case 'W':
+			{
+				masterLevel = Logging::GetNamedLevel(optarg);
+				if (masterLevel == Log::INVALID)
+				{
+					BOX_FATAL("Invalid logging level");
+					return 2;
+				}
+			}
+			break;
+
 		case 'w':
 			// Read/write mode
 			readWrite = true;
@@ -197,6 +204,13 @@
 	
 	Logging::SetGlobalLevel((Log::Level)masterLevel);
 
+	bool quiet = false;
+	if (masterLevel < Log::NOTICE)
+	{
+		// Quiet mode
+		quiet = true;
+	}
+
 	// Print banner?
 	if(!quiet)
 	{

Modified: box/trunk/test/backupstorefix/testbackupstorefix.cpp
===================================================================
--- box/trunk/test/backupstorefix/testbackupstorefix.cpp	2008-05-28 15:28:36 UTC (rev 2183)
+++ box/trunk/test/backupstorefix/testbackupstorefix.cpp	2008-05-28 15:35:42 UTC (rev 2184)
@@ -338,7 +338,7 @@
 		}
 		
 		// Generate a list of all the object IDs
-		TEST_THAT_ABORTONFAIL(::system(BBACKUPQUERY " -q "
+		TEST_THAT_ABORTONFAIL(::system(BBACKUPQUERY " -Wwarning "
 			"-c testfiles/bbackupd.conf \"list -r\" quit "
 			"> testfiles/initial-listing.txt") == 0);
 

Modified: box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in
===================================================================
--- box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in	2008-05-28 15:28:36 UTC (rev 2183)
+++ box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in	2008-05-28 15:35:42 UTC (rev 2184)
@@ -93,8 +93,12 @@
 	}
 
 	# read in the new listing, and compare
-	open LISTING,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit |" or die "Can't open list utility";
-	open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file";
+	open LISTING,"../../bin/bbackupquery/bbackupquery -Wwarning " .
+		"-c testfiles/bbackupd.conf " .
+		"\"list -r\" quit |"
+		or die "Can't open list utility";
+	open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt'
+		or die "can't open copy listing file";
 	my $err = 0;
 	while(<LISTING>)
 	{
@@ -125,8 +129,12 @@
 }
 elsif($ARGV[0] eq 'reroot')
 {
-	open LISTING,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit |" or die "Can't open list utility";
-	open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file";
+	open LISTING,"../../bin/bbackupquery/bbackupquery -Wwarning " .
+		"-c testfiles/bbackupd.conf " .
+		"\"list -r\" quit |"
+		or die "Can't open list utility";
+	open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt'
+		or die "can't open copy listing file";
 	my $err = 0;
 	my $count = 0;
 	while(<LISTING>)

Modified: box/trunk/test/bbackupd/testbbackupd.cpp
===================================================================
--- box/trunk/test/bbackupd/testbbackupd.cpp	2008-05-28 15:28:36 UTC (rev 2183)
+++ box/trunk/test/bbackupd/testbbackupd.cpp	2008-05-28 15:35:42 UTC (rev 2184)
@@ -43,6 +43,7 @@
 #include "BackupClientRestore.h"
 #include "BackupDaemon.h"
 #include "BackupDaemonConfigVerify.h"
+#include "BackupQueries.h"
 #include "BackupStoreConstants.h"
 #include "BackupStoreDirectory.h"
 #include "BackupStoreException.h"
@@ -82,13 +83,21 @@
 // utility macro for comparing two strings in a line
 #define TEST_EQUAL(expected, found, line) \
 { \
-	std::string exp_str = expected; \
-	std::string found_str = found; \
+	std::ostringstream _oss1; \
+	_oss1 << expected; \
+	std::string exp_str = _oss1.str(); \
+	\
+	std::ostringstream _oss2; \
+	_oss2 << found; \
+	std::string found_str = _oss2.str(); \
+	\
 	TEST_THAT(exp_str == found_str); \
+	std::string _line = line; \
+	\
 	if(exp_str != found_str) \
 	{ \
 		printf("Expected <%s> but found <%s> in <%s>\n", \
-			exp_str.c_str(), found_str.c_str(), line.c_str()); \
+			exp_str.c_str(), found_str.c_str(), _line.c_str()); \
 	} \
 }
 
@@ -336,7 +345,11 @@
 #endif
 	
 	BackupClientFileAttributes t3;
-	TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"), CommonException, OSFileError);
+	{
+		Logging::Guard guard(Log::ERROR);
+		TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"),
+			CommonException, OSFileError);
+	}
 
 	// Create some more files
 	FILE *f = fopen("testfiles/test1_n", "w");
@@ -353,8 +366,13 @@
 #endif
 
 #ifndef WIN32
-	TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"), CommonException, OSFileError);
-	TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"), BackupStoreException, AttributesNotLoaded);
+	{
+		Logging::Guard guard(Log::ERROR);
+		TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"),
+			CommonException, OSFileError);
+		TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"),
+			BackupStoreException, AttributesNotLoaded);
+	}
 
 	// Test that attributes are vaguely similar
 	TEST_THAT(attrmatch("testfiles/test1", "testfiles/test1_n"));
@@ -650,6 +668,7 @@
 {
 	// ensure that no child processes end up running tests!
 	int own_pid = getpid();
+	BOX_TRACE("Test PID is " << own_pid);
 		
 	// this is a quick hack to allow passing some options to the daemon
 	const char* argv[] = {
@@ -669,14 +688,10 @@
 		result = daemon.Main("testfiles/bbackupd.conf", 1, argv);
 	}
 	
-	TEST_THAT(result == 0);
-	if (result != 0)
-	{
-		printf("Daemon exited with code %d\n", result);
-	}
+	TEST_EQUAL(0, result, "Daemon exit code");
 	
 	// ensure that no child processes end up running tests!
-	TEST_THAT(getpid() == own_pid);
+	TEST_EQUAL(own_pid, getpid(), "Forking test problem");
 	if (getpid() != own_pid)
 	{
 		// abort!
@@ -799,12 +814,40 @@
 int lstat_test_hook(const char *file_name, struct stat *buf)
 #endif
 {
-	// TRACE1("lstat hook triggered for %s", file_name);		
+	// TRACE1("lstat hook triggered for %s", file_name);
 	memset(buf, 0, sizeof(*buf));
 	buf->st_mode = S_IFREG;
 	return 0;
 }
 
+// Simulate a symlink that is on a different device than the file
+// that it points to.
+int lstat_test_post_hook(int old_ret, const char *file_name, struct stat *buf)
+{
+	BOX_TRACE("lstat post hook triggered for " << file_name);
+	if (old_ret == 0 &&
+		strcmp(file_name, "testfiles/symlink-to-TestDir1") == 0)
+	{
+		buf->st_dev ^= 0xFFFF;
+	}
+	return old_ret;
+}
+
+bool test_entry_deleted(BackupStoreDirectory& rDir, 
+	const std::string& rName)
+{
+	BackupStoreDirectory::Iterator i(rDir);
+
+	BackupStoreDirectory::Entry *en = i.FindMatchingClearName(
+		BackupStoreFilenameClear(rName));
+	TEST_THAT(en != 0);
+	if (en == 0) return false;
+
+	int16_t flags = en->GetFlags();
+	TEST_THAT(flags && BackupStoreDirectory::Entry::Flags_Deleted);
+	return flags && BackupStoreDirectory::Entry::Flags_Deleted;
+}
+
 int test_bbackupd()
 {
 	// First, wait for a normal period to make sure the last changes 
@@ -818,6 +861,23 @@
 			"testfiles/clientPrivKey.pem",
 			"testfiles/clientTrustedCAs.pem");
 
+	printf("\n==== Testing that ReadDirectory on nonexistent directory "
+		"does not crash\n");
+	{
+		std::auto_ptr<BackupProtocolClient> client = ConnectAndLogin(
+			context, 0 /* read-write */);
+		
+		{
+			Logging::Guard guard(Log::ERROR);
+			TEST_CHECK_THROWS(ReadDirectory(*client, 0x12345678),
+				ConnectionException,
+				Conn_Protocol_UnexpectedReply);
+		}
+
+		client->QueryFinished();
+		sSocket.Close();
+	}
+
 	// unpack the files for the initial test
 	TEST_THAT(::system("rm -rf testfiles/TestDir1") == 0);
 	TEST_THAT(::mkdir("testfiles/TestDir1", 0777) == 0);
@@ -852,7 +912,8 @@
 		char buffer[10000];
 		memset(buffer, 0, sizeof(buffer));
 
-		TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer));
+		TEST_EQUAL(sizeof(buffer), write(fd, buffer, sizeof(buffer)),
+			"Buffer write");
 		TEST_THAT(close(fd) == 0);
 		
 		int pid = start_internal_daemon();
@@ -867,11 +928,13 @@
 		TEST_THAT(unlink("testfiles/bbackupd.log") == 0);
 
 		pid = start_internal_daemon();
+		intercept_clear_setup();
 		
 		fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY);
 		TEST_THAT(fd > 0);
 		// write again, to update the file's timestamp
-		TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer));
+		TEST_EQUAL(sizeof(buffer), write(fd, buffer, sizeof(buffer)),
+			"Buffer write");
 		TEST_THAT(close(fd) == 0);	
 
 		wait_for_backup_operation();
@@ -937,11 +1000,13 @@
 		intercept_setup_delay("testfiles/TestDir1/spacetest/f1", 
 			0, 4000, SYS_read, 1);
 		pid = start_internal_daemon();
+		intercept_clear_setup();
 		
 		fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY);
 		TEST_THAT(fd > 0);
 		// write again, to update the file's timestamp
-		TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer));
+		TEST_EQUAL(sizeof(buffer), write(fd, buffer, sizeof(buffer)),
+			"Buffer write");
 		TEST_THAT(close(fd) == 0);	
 
 		wait_for_backup_operation();
@@ -970,9 +1035,9 @@
 			std::string line;
 			TEST_THAT(reader.GetLine(line));
 			std::string comp = "Receive Success(0x";
-			TEST_THAT(line.substr(0, comp.size()) == comp);
+			TEST_EQUAL(comp, line.substr(0, comp.size()), line);
 			TEST_THAT(reader.GetLine(line));
-			TEST_THAT(line == "Receiving stream, size 124");
+			TEST_EQUAL("Receiving stream, size 124", line, line);
 
 			// delaying for 4 seconds in one step means that
 			// the diff timer and the keepalive timer will
@@ -997,11 +1062,13 @@
 		intercept_setup_delay("testfiles/TestDir1/spacetest/f1", 
 			0, 1000, SYS_read, 3);
 		pid = start_internal_daemon();
+		intercept_clear_setup();
 		
 		fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY);
 		TEST_THAT(fd > 0);
 		// write again, to update the file's timestamp
-		TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer));
+		TEST_EQUAL(sizeof(buffer), write(fd, buffer, sizeof(buffer)),
+			"Buffer write");
 		TEST_THAT(close(fd) == 0);	
 
 		wait_for_backup_operation();
@@ -1030,9 +1097,9 @@
 			std::string line;
 			TEST_THAT(reader.GetLine(line));
 			std::string comp = "Receive Success(0x";
-			TEST_THAT(line.substr(0, comp.size()) == comp);
+			TEST_EQUAL(comp, line.substr(0, comp.size()), line);
 			TEST_THAT(reader.GetLine(line));
-			TEST_THAT(line == "Receiving stream, size 124");
+			TEST_EQUAL("Receiving stream, size 124", line, line);
 
 			// delaying for 3 seconds in steps of 1 second
 			// means that the keepalive timer will expire 3 times,
@@ -1041,13 +1108,13 @@
 			// only two keepalives.
 			
 			TEST_THAT(reader.GetLine(line));
-			TEST_THAT(line == "Send GetIsAlive()");
+			TEST_EQUAL("Send GetIsAlive()", line, line);
 			TEST_THAT(reader.GetLine(line));
-			TEST_THAT(line == "Receive IsAlive()");
+			TEST_EQUAL("Receive IsAlive()", line, line);
 			TEST_THAT(reader.GetLine(line));
-			TEST_THAT(line == "Send GetIsAlive()");
+			TEST_EQUAL("Send GetIsAlive()", line, line);
 			TEST_THAT(reader.GetLine(line));
-			TEST_THAT(line == "Receive IsAlive()");
+			TEST_EQUAL("Receive IsAlive()", line, line);
 
 			// but two matching blocks should have been found
 			// already, so the upload should be a diff.
@@ -1078,6 +1145,7 @@
 		readdir_stop_time = time(NULL) + 12 + 2;
 
 		pid = start_internal_daemon();
+		intercept_clear_setup();
 		
 		std::string touchfile = 
 			"testfiles/TestDir1/spacetest/d1/touch-me";
@@ -1085,7 +1153,8 @@
 		fd = open(touchfile.c_str(), O_CREAT | O_WRONLY);
 		TEST_THAT(fd > 0);
 		// write again, to update the file's timestamp
-		TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer));
+		TEST_EQUAL(sizeof(buffer), write(fd, buffer, sizeof(buffer)),
+			"Buffer write");
 		TEST_THAT(close(fd) == 0);	
 
 		wait_for_backup_operation();
@@ -1158,7 +1227,7 @@
 #endif // PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
 
 	std::string cmd = BBACKUPD " " + bbackupd_args + 
-		" testfiles/bbackupd-temploc.conf";
+		" testfiles/bbackupd.conf";
 
 	bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
 	TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
@@ -1169,6 +1238,368 @@
 	if (!ServerIsAlive(bbackupd_pid)) return 1;
 	if (!ServerIsAlive(bbstored_pid)) return 1;
 
+	if(bbackupd_pid > 0)
+	{
+		printf("\n==== Testing that backup pauses when "
+			"store is full\n");
+
+		// wait for files to be uploaded
+		BOX_TRACE("Waiting for all outstanding files to be uploaded")
+		wait_for_sync_end();
+		BOX_TRACE("done.")
+
+		// Set limit to something very small
+		// 26 blocks will be used at this point.
+		// (12 files + location * 2 for raidfile)
+		// 20 is what we'll need in a minute
+		// set soft limit to 0 to ensure that all deleted files
+		// are deleted immediately by housekeeping
+		TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c "
+			"testfiles/bbstored.conf setlimit 01234567 0B 20B") 
+			== 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+		// Unpack some more files
+		#ifdef WIN32
+			TEST_THAT(::system("tar xzvf testfiles/spacetest2.tgz "
+				"-C testfiles/TestDir1") == 0);
+		#else
+			TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz "
+				"| ( cd testfiles/TestDir1 && tar xf - )") == 0);
+		#endif
+
+		// Delete a file and a directory
+		TEST_THAT(::unlink("testfiles/TestDir1/spacetest/f1") == 0);
+		TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d7") == 0);
+
+		// The following files should be on the server:
+		// 00000001 -d---- 00002 (root)
+		// 00000002 -d---- 00002 Test1
+		// 00000003 -d---- 00002 Test1/spacetest
+		// 00000004 f-X--- 00002 Test1/spacetest/f1
+		// 00000005 f----- 00002 Test1/spacetest/f2
+		// 00000006 -d---- 00002 Test1/spacetest/d1
+		// 00000007 f----- 00002 Test1/spacetest/d1/f3
+		// 00000008 f----- 00002 Test1/spacetest/d1/f4
+		// 00000009 -d---- 00002 Test1/spacetest/d2
+		// 0000000a -d---- 00002 Test1/spacetest/d3
+		// 0000000b -d---- 00002 Test1/spacetest/d3/d4
+		// 0000000c f----- 00002 Test1/spacetest/d3/d4/f5
+		// 0000000d -d---- 00002 Test1/spacetest/d6
+		// 0000000e -dX--- 00002 Test1/spacetest/d7
+		// This is 28 blocks total, of which 2 in deleted files
+		// and 18 in directories. Note that f1 and d7 may or may
+		// not be deleted yet.
+		//
+		// spacetest1 + spacetest2 = 16 files = 32 blocks with raidfile
+		// minus one file and one dir is 28 blocks
+		//
+		// d2/f6, d6/d8 and d6/d8/f7 are new
+		// even if the client marks f1 and d7 as deleted, and
+		// housekeeping deleted them, the backup cannot complete
+		// if the limit is 20 blocks.
+
+		BOX_TRACE("Waiting for bbackupd to notice that the "
+			"store is full");
+		wait_for_sync_end();
+		BOX_TRACE("done.");
+
+		BOX_TRACE("Compare to check that there are differences");
+		int compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query0a.log "
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		BOX_TRACE("done.");
+
+		// Check that the notify script was run
+		TEST_THAT(TestFileExists("testfiles/notifyran.store-full.1"));
+		// But only once!
+		TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2"));
+		
+		// Kill the daemon
+		terminate_bbackupd(bbackupd_pid);
+
+		BOX_TRACE("Wait for housekeeping to remove the deleted files");
+		wait_for_backup_operation(5);
+		BOX_TRACE("done.");
+
+		// This removes f1 and d7, which were previously marked
+		// as deleted, so total usage drops by 4 blocks to 24.
+
+		// BLOCK
+		{
+			std::auto_ptr<BackupProtocolClient> client =
+				ConnectAndLogin(context, 0 /* read-write */);
+		
+			std::auto_ptr<BackupProtocolClientAccountUsage> usage(
+				client->QueryGetAccountUsage());
+			TEST_EQUAL(24, usage->GetBlocksUsed(), "blocks used");
+			TEST_EQUAL(0,  usage->GetBlocksInDeletedFiles(),
+				"deleted blocks");
+			TEST_EQUAL(16, usage->GetBlocksInDirectories(),
+				"directory blocks");
+
+			client->QueryFinished();
+			sSocket.Close();
+		}
+
+		BOX_TRACE("Restart bbackupd with more exclusions");
+		// Start again with a new config that excludes d3 and f2,
+		// and hence also d3/d4 and d3/d4/f5. bbackupd should mark
+		// them as deleted and housekeeping should clean up,
+		// making space to upload the new files.
+		// total required: (13-2-4+3)*2 = 20 blocks
+		/*
+		cmd = BBACKUPD " " + bbackupd_args +
+			" testfiles/bbackupd-exclude.conf";
+		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
+		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
+		::safe_sleep(1);
+		TEST_THAT(ServerIsAlive(bbackupd_pid));
+		TEST_THAT(ServerIsAlive(bbstored_pid));
+		if (!ServerIsAlive(bbackupd_pid)) return 1;
+		if (!ServerIsAlive(bbstored_pid)) return 1;
+		*/
+		BackupDaemon bbackupd;
+		bbackupd.Configure("testfiles/bbackupd-exclude.conf");
+		bbackupd.InitCrypto();
+		BOX_TRACE("done.");
+
+		// Should be marked as deleted by this run
+		// wait_for_sync_end();
+		{
+			Logging::Guard guard(Log::ERROR);
+			bbackupd.RunSyncNow();
+		}
+
+		TEST_THAT(bbackupd.StorageLimitExceeded());
+
+		// Check that the notify script was run
+		// TEST_THAT(TestFileExists("testfiles/notifyran.store-full.2"));
+		// But only twice!
+		// TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.3"));
+
+		// All these should be marked as deleted but hopefully
+		// not removed by housekeeping yet:
+		// f1		deleted
+		// f2		excluded
+		// d1		excluded (why?)
+		// d1/f3	excluded (why?)
+		// d3		excluded
+		// d3/d4	excluded
+		// d3/d4/f5	excluded
+		// d7		deleted
+		// Careful with timing here, these files can already be
+		// deleted by housekeeping.
+
+		BOX_TRACE("Find out whether bbackupd marked files as deleted");
+		{
+			std::auto_ptr<BackupProtocolClient> client =
+				ConnectAndLogin(context, 0 /* read-write */);
+		
+			std::auto_ptr<BackupStoreDirectory> rootDir = 
+				ReadDirectory(*client,
+				BackupProtocolClientListDirectory::RootDirectory);
+
+			int64_t testDirId = SearchDir(*rootDir, "Test1");
+			TEST_THAT(testDirId != 0);
+
+			std::auto_ptr<BackupStoreDirectory> Test1_dir =
+				ReadDirectory(*client, testDirId);
+
+			int64_t spacetestDirId = SearchDir(*Test1_dir, 
+				"spacetest");
+			TEST_THAT(spacetestDirId != 0);
+
+			std::auto_ptr<BackupStoreDirectory> spacetest_dir =
+				ReadDirectory(*client, spacetestDirId);
+
+			TEST_THAT(SearchDir(*spacetest_dir, "f1") == 0);
+			TEST_THAT(test_entry_deleted(*spacetest_dir, "f2"));
+			TEST_THAT(test_entry_deleted(*spacetest_dir, "d3"));
+			TEST_THAT(SearchDir(*spacetest_dir, "d7") == 0);
+
+			int64_t d3_id = SearchDir(*spacetest_dir, "d3");
+			TEST_THAT(d3_id != 0);
+
+			std::auto_ptr<BackupStoreDirectory> d3_dir =
+				ReadDirectory(*client, d3_id);
+			TEST_THAT(test_entry_deleted(*d3_dir, "d4"));
+
+			int64_t d4_id = SearchDir(*d3_dir, "d4");
+			TEST_THAT(d4_id != 0);
+
+			std::auto_ptr<BackupStoreDirectory> d4_dir =
+				ReadDirectory(*client, d4_id);
+			TEST_THAT(test_entry_deleted(*d4_dir, "f5"));
+
+			std::auto_ptr<BackupProtocolClientAccountUsage> usage(
+				client->QueryGetAccountUsage());
+			TEST_EQUAL(24, usage->GetBlocksUsed(), "blocks used");
+			TEST_EQUAL(4, usage->GetBlocksInDeletedFiles(),
+				"deleted blocks");
+			TEST_EQUAL(16, usage->GetBlocksInDirectories(),
+				"directory blocks");
+			// d1/f3 and d1/f4 are the only two files on the
+			// server which are not deleted, they use 2 blocks
+			// each, the rest is directories and 2 deleted files
+			// (f1 and d3/d4/f5)
+
+			// Log out.
+			client->QueryFinished();
+			sSocket.Close();
+		}
+		BOX_TRACE("done.");
+
+		// Wait for housekeeping to run
+		BOX_TRACE("Wait for housekeeping to remove the deleted files");
+		wait_for_backup_operation(5);
+		BOX_TRACE("done.");
+
+		BOX_TRACE("Check that the files were removed");
+		{
+			std::auto_ptr<BackupProtocolClient> client = 
+				ConnectAndLogin(context, 0 /* read-write */);
+			
+			std::auto_ptr<BackupStoreDirectory> rootDir = 
+				ReadDirectory(*client,
+				BackupProtocolClientListDirectory::RootDirectory);
+
+			int64_t testDirId = SearchDir(*rootDir, "Test1");
+			TEST_THAT(testDirId != 0);
+
+			std::auto_ptr<BackupStoreDirectory> Test1_dir =
+				ReadDirectory(*client, testDirId);
+
+			int64_t spacetestDirId = SearchDir(*Test1_dir, 
+				"spacetest");
+			TEST_THAT(spacetestDirId != 0);
+
+			std::auto_ptr<BackupStoreDirectory> spacetest_dir =
+				ReadDirectory(*client, spacetestDirId);
+
+			TEST_THAT(SearchDir(*spacetest_dir, "f1") == 0);
+			TEST_THAT(SearchDir(*spacetest_dir, "f2") == 0);
+			TEST_THAT(SearchDir(*spacetest_dir, "d3") == 0);
+			TEST_THAT(SearchDir(*spacetest_dir, "d7") == 0);
+
+			std::auto_ptr<BackupProtocolClientAccountUsage> usage(
+				client->QueryGetAccountUsage());
+			TEST_EQUAL(16, usage->GetBlocksUsed(), "blocks used");
+			TEST_EQUAL(0, usage->GetBlocksInDeletedFiles(),
+				"deleted blocks");
+			TEST_EQUAL(12, usage->GetBlocksInDirectories(),
+				"directory blocks");
+			// d1/f3 and d1/f4 are the only two files on the
+			// server, they use 2 blocks each, the rest is
+			// directories.
+
+			// Log out.
+			client->QueryFinished();
+			sSocket.Close();
+		}
+
+		// Need 22 blocks free to upload everything
+		TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c "
+			"testfiles/bbstored.conf setlimit 01234567 0B 22B") 
+			== 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+		// Run another backup, now there should be enough space
+		// for everything we want to upload.
+		{
+			Logging::Guard guard(Log::ERROR);
+			bbackupd.RunSyncNow();
+		}
+		TEST_THAT(!bbackupd.StorageLimitExceeded());
+
+		// Check that the contents of the store are the same 
+		// as the contents of the disc 
+		// (-a = all, -c = give result in return code)
+		BOX_TRACE("Check that all files were uploaded successfully");
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd-exclude.conf "
+			"-l testfiles/query1.log "
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		BOX_TRACE("done.");
+
+		// BLOCK
+		{
+			std::auto_ptr<BackupProtocolClient> client = 
+				ConnectAndLogin(context, 0 /* read-write */);
+
+			std::auto_ptr<BackupProtocolClientAccountUsage> usage(
+				client->QueryGetAccountUsage());
+			TEST_EQUAL(22, usage->GetBlocksUsed(), "blocks used");
+			TEST_EQUAL(0, usage->GetBlocksInDeletedFiles(),
+				"deleted blocks");
+			TEST_EQUAL(14, usage->GetBlocksInDirectories(),
+				"directory blocks");
+			// d2/f6, d6/d8 and d6/d8/f7 are new
+			// i.e. 2 new files, 1 new directory
+
+			client->QueryFinished();
+			sSocket.Close();
+		}
+
+		// Put the limit back
+		TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c "
+			"testfiles/bbstored.conf setlimit 01234567 "
+			"1000B 2000B") == 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+	
+		// Start again with the old config
+		BOX_TRACE("Restart bbackupd with original configuration");
+		// terminate_bbackupd();
+		cmd = BBACKUPD " " + bbackupd_args +
+			" testfiles/bbackupd.conf";
+		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
+		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
+		::safe_sleep(1);
+		TEST_THAT(ServerIsAlive(bbackupd_pid));
+		TEST_THAT(ServerIsAlive(bbstored_pid));
+		if (!ServerIsAlive(bbackupd_pid)) return 1;
+		if (!ServerIsAlive(bbstored_pid)) return 1;
+		BOX_TRACE("done.");
+
+		// unpack the initial files again
+		#ifdef WIN32
+			TEST_THAT(::system("tar xzvf testfiles/test_base.tgz "
+				"-C testfiles") == 0);
+		#else
+			TEST_THAT(::system("gzip -d < testfiles/test_base.tgz "
+				"| ( cd testfiles && tar xf - )") == 0);
+		#endif
+
+		BOX_TRACE("Wait for bbackupd to upload more files");
+		wait_for_backup_operation();
+		BOX_TRACE("done.");
+		
+		// Check that the contents of the store are the same 
+		// as the contents of the disc 
+		// (-a = all, -c = give result in return code)
+		BOX_TRACE("Check that all files were uploaded successfully");
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query1.log "
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue, 
+			BackupQueries::ReturnCode::Compare_Same);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		BOX_TRACE("done.");
+
+		TEST_THAT(ServerIsAlive(bbackupd_pid));
+		TEST_THAT(ServerIsAlive(bbstored_pid));
+		if (!ServerIsAlive(bbackupd_pid)) return 1;
+		if (!ServerIsAlive(bbstored_pid)) return 1;
+	}
+
 	#ifndef WIN32
 	printf("\n==== Testing that absolute symlinks are not followed "
 		"during restore\n");
@@ -1204,11 +1635,12 @@
 		::sync_and_wait();
 
 		// Check that the backup was successful, i.e. no differences
-		int compareReturnValue = ::system(BBACKUPQUERY " -q "
+		int compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query1.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// now stop bbackupd and update the test file,
@@ -1227,9 +1659,10 @@
 		// check that we can restore it
 		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
-			"-q \"restore Test1 testfiles/restore-symlink\" "
+			"-Wwarning \"restore Test1 testfiles/restore-symlink\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Command_OK);
 
 		// make it accessible again
 		TEST_THAT(chmod(SYM_DIR, 0755) == 0);
@@ -1240,10 +1673,52 @@
 		std::string line;
 		TEST_THAT(gl.GetLine(line));
 		TEST_THAT(line != "before");
-		TEST_THAT(line == "after");
+		TEST_EQUAL("after", line, line);
 
 		#undef SYM_DIR
 
+		/*
+ 		// This is not worth testing or fixing.
+ 		//
+		#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
+		printf("\n==== Testing that symlinks to other filesystems "
+			"can be backed up as roots\n");
+
+		intercept_setup_lstat_post_hook(lstat_test_post_hook);
+		TEST_THAT(symlink("TestDir1", "testfiles/symlink-to-TestDir1")
+			== 0);
+
+		struct stat stat_st, lstat_st;
+		TEST_THAT(stat("testfiles/symlink-to-TestDir1", &stat_st) == 0);
+		TEST_THAT(lstat("testfiles/symlink-to-TestDir1", &lstat_st) == 0);
+		TEST_EQUAL((stat_st.st_dev ^ 0xFFFF), lstat_st.st_dev,
+			"stat vs lstat");
+
+		BackupDaemon bbackupd;
+		bbackupd.Configure("testfiles/bbackupd-symlink.conf");
+		bbackupd.InitCrypto();
+		bbackupd.RunSyncNow();
+		intercept_clear_setup();
+
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query0a.log "
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// and again using the symlink during compare
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd-symlink.conf "
+			"-l testfiles/query0a.log "
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		#endif
+		*/
+
 		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
 		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
 		::safe_sleep(1);
@@ -1257,7 +1732,27 @@
 
 	printf("\n==== Testing that redundant locations are deleted on time\n");
 
+	// BLOCK
 	{
+		// Kill the daemon
+		terminate_bbackupd(bbackupd_pid);
+
+		// Start it with a config that has a temporary location
+		// that will be created on the server
+		std::string cmd = BBACKUPD " " + bbackupd_args + 
+			" testfiles/bbackupd-temploc.conf";
+
+		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
+		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
+		::safe_sleep(1);
+
+		TEST_THAT(ServerIsAlive(bbackupd_pid));
+		TEST_THAT(ServerIsAlive(bbstored_pid));
+		if (!ServerIsAlive(bbackupd_pid)) return 1;
+		if (!ServerIsAlive(bbstored_pid)) return 1;
+
+		sync_and_wait();
+
 		{
 			std::auto_ptr<BackupProtocolClient> client =
 				ConnectAndLogin(context,
@@ -1266,16 +1761,9 @@
 			std::auto_ptr<BackupStoreDirectory> dir = 
 				ReadDirectory(*client,
 				BackupProtocolClientListDirectory::RootDirectory);
-
-			// int64_t testDirId = SearchDir(*dir, "Test2");
-			// TEST_THAT(testDirId == 0);
-
-			sync_and_wait();
-
-			dir = ReadDirectory(*client,
-				BackupProtocolClientListDirectory::RootDirectory);
 			int64_t testDirId = SearchDir(*dir, "Test2");
 			TEST_THAT(testDirId != 0);
+
 			client->QueryFinished();
 			sSocket.Close();
 		}
@@ -1283,6 +1771,7 @@
 		// Kill the daemon
 		terminate_bbackupd(bbackupd_pid);
 
+		// Start it again with the normal config (no Test2)
 		cmd = BBACKUPD " " + bbackupd_args +
 			" testfiles/bbackupd.conf";
 		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
@@ -1301,6 +1790,8 @@
 		wait_for_sync_end();
 		wait_for_sync_end();
 
+		// not yet! should still be there
+
 		{
 			std::auto_ptr<BackupProtocolClient> client =
 				ConnectAndLogin(context,
@@ -1311,30 +1802,26 @@
 				BackupProtocolClientListDirectory::RootDirectory);
 			int64_t testDirId = SearchDir(*dir, "Test2");
 			TEST_THAT(testDirId != 0);
+
 			client->QueryFinished();
 			sSocket.Close();
 		}
 
 		wait_for_sync_end();
 
+		// NOW it should be gone
+
 		{
 			std::auto_ptr<BackupProtocolClient> client =
 				ConnectAndLogin(context,
 				BackupProtocolClientLogin::Flags_ReadOnly);
 			
-			std::auto_ptr<BackupStoreDirectory> dir = 
+			std::auto_ptr<BackupStoreDirectory> root_dir = 
 				ReadDirectory(*client,
 				BackupProtocolClientListDirectory::RootDirectory);
-			int64_t testDirId = SearchDir(*dir, "Test2");
-			TEST_THAT(testDirId != 0);
 
-			BackupStoreDirectory::Iterator i(*dir);
-			BackupStoreFilenameClear dirname("Test2");
-			BackupStoreDirectory::Entry *en = 
-				i.FindMatchingClearName(dirname);
-			TEST_THAT(en != 0);
-			int16_t en_flags = en->GetFlags();
-			TEST_THAT(en_flags && BackupStoreDirectory::Entry::Flags_Deleted);
+			TEST_THAT(test_entry_deleted(*root_dir, "Test2"));
+
 			client->QueryFinished();
 			sSocket.Close();
 		}
@@ -1347,82 +1834,6 @@
 
 	if(bbackupd_pid > 0)
 	{
-		printf("\n==== Testing that backup pauses when store is full\n");
-
-		// wait for files to be uploaded
-		wait_for_backup_operation();
-
-		// Set limit to something very small
-		// About 28 blocks will be used at this point. bbackupd 
-		// will only pause if the size used is greater than 
-		// soft limit + 1/3 of (hard - soft). Set small values
-		// for limits accordingly.
-		TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c "
-			"testfiles/bbstored.conf setlimit 01234567 9B 10B") 
-			== 0);
-		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
-
-		// Unpack some more files
-		#ifdef WIN32
-			TEST_THAT(::system("tar xzvf testfiles/spacetest2.tgz "
-				"-C testfiles/TestDir1") == 0);
-		#else
-			TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz "
-				"| ( cd testfiles/TestDir1 && tar xf - )") == 0);
-		#endif
-
-		// Delete a file and a directory
-		TEST_THAT(::unlink("testfiles/TestDir1/spacetest/d1/f3") == 0);
-		TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d3/d4") == 0);
-		wait_for_backup_operation();
-
-		// Make sure there are some differences
-		int compareReturnValue = ::system(BBACKUPQUERY " -q "
-			"-c testfiles/bbackupd.conf "
-			"-l testfiles/query0a.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 2);
-		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-
-		// Put the limit back
-		TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c "
-			"testfiles/bbstored.conf setlimit 01234567 "
-			"1000B 2000B") == 0);
-		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
-		
-		// Check that the notify script was run
-		TEST_THAT(TestFileExists("testfiles/notifyran.store-full.1"));
-		// But only once!
-		TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2"));
-		
-		// unpack the initial files again
-		#ifdef WIN32
-			TEST_THAT(::system("tar xzvf testfiles/test_base.tgz "
-				"-C testfiles") == 0);
-		#else
-			TEST_THAT(::system("gzip -d < testfiles/test_base.tgz "
-				"| ( cd testfiles && tar xf - )") == 0);
-		#endif
-
-		// wait for it to do it's stuff
-		wait_for_backup_operation();
-		
-		// Check that the contents of the store are the same 
-		// as the contents of the disc 
-		// (-a = all, -c = give result in return code)
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
-			"-c testfiles/bbackupd.conf "
-			"-l testfiles/query1.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
-		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-
-		TEST_THAT(ServerIsAlive(bbackupd_pid));
-		TEST_THAT(ServerIsAlive(bbstored_pid));
-		if (!ServerIsAlive(bbackupd_pid)) return 1;
-		if (!ServerIsAlive(bbstored_pid)) return 1;
-
-
 		printf("\n==== Check that read-only directories and "
 			"their contents can be restored.\n");
 
@@ -1438,27 +1849,33 @@
 			wait_for_sync_end(); // too new
 			wait_for_sync_end(); // should be backed up now
 
-			compareReturnValue = ::system(BBACKUPQUERY " "
+			int compareReturnValue = ::system(BBACKUPQUERY " "
+				"-Wwarning "
 				"-c testfiles/bbackupd.conf "
-				"-q \"compare -cEQ Test1 testfiles/TestDir1\" " 
+				"\"compare -cEQ Test1 testfiles/TestDir1\" " 
 				"quit");
-			TEST_RETURN(compareReturnValue, 1);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// check that we can restore it
 			compareReturnValue = ::system(BBACKUPQUERY " "
+				"-Wwarning "
 				"-c testfiles/bbackupd.conf "
-				"-q \"restore Test1 testfiles/restore1\" "
+				"\"restore Test1 testfiles/restore1\" "
 				"quit");
-			TEST_RETURN(compareReturnValue, 0);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Command_OK);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// check that it restored properly
 			compareReturnValue = ::system(BBACKUPQUERY " "
+				"-Wwarning "
 				"-c testfiles/bbackupd.conf "
-				"-q \"compare -cEQ Test1 testfiles/restore1\" " 
+				"\"compare -cEQ Test1 testfiles/restore1\" " 
 				"quit");
-			TEST_RETURN(compareReturnValue, 1);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// put the permissions back to sensible values
@@ -1513,7 +1930,7 @@
 		std::string filepath(dirpath + "/" + filename);
 
 		char cwdbuf[1024];
-		TEST_THAT(getcwd(cwdbuf, sizeof(cwdbuf)) == cwdbuf);
+		TEST_EQUAL(cwdbuf, getcwd(cwdbuf, sizeof(cwdbuf)), "getcwd");
 		std::string cwd = cwdbuf;
 
 		// Test that our emulated chdir() works properly
@@ -1544,52 +1961,61 @@
 
 		// test that bbackupd will let us lcd into the local 
 		// directory using a relative path
-		std::string command = BBACKUPQUERY " -q "
+		std::string command = BBACKUPQUERY " "
+			"-Wwarning "
 			"-c testfiles/bbackupd.conf "
 			"\"lcd testfiles/TestDir1/" + systemDirName + "\" "
 			"quit";
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Command_OK);
 
 		// and back out again
-		command = BBACKUPQUERY " -q "
+		command = BBACKUPQUERY " "
+			"-Wwarning "
 			"-c testfiles/bbackupd.conf "
 			"\"lcd testfiles/TestDir1/" + systemDirName + "\" "
 			"\"lcd ..\" quit";
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Command_OK);
 
 		// and using an absolute path
-		command = BBACKUPQUERY " -q "
+		command = BBACKUPQUERY " "
+			"-Wwarning "
 			"-c testfiles/bbackupd.conf "
 			"\"lcd " + cwd + "/testfiles/TestDir1/" + 
 			systemDirName + "\" quit";
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Command_OK);
 
 		// and back out again
-		command = BBACKUPQUERY " -q "
+		command = BBACKUPQUERY " "
+			"-Wwarning "
 			"-c testfiles/bbackupd.conf "
 			"\"lcd " + cwd + "/testfiles/TestDir1/" + 
 			systemDirName + "\" "
 			"\"lcd ..\" quit";
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Command_OK);
 
 		{
 			FileStream fs(filepath.c_str(), O_CREAT | O_RDWR);
 
 			std::string data("hello world\n");
 			fs.Write(data.c_str(), data.size());
-			TEST_THAT(fs.GetPosition() == 12);
+			TEST_EQUAL(12, fs.GetPosition(), "FileStream position");
 			fs.Close();
 		}
 
 		wait_for_backup_operation();
 		// Compare to check that the file was uploaded
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " -Wwarning "
 			"-c testfiles/bbackupd.conf \"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// Check that we can find it in directory listing
@@ -1615,9 +2041,8 @@
 			sSocket.Close();
 		}
 
-
 		// Check that bbackupquery shows the dir in console encoding
-		command = BBACKUPQUERY " -q "
+		command = BBACKUPQUERY " -Wwarning "
 			"-c testfiles/bbackupd.conf "
 			"-q \"list Test1\" quit";
 		pid_t bbackupquery_pid;
@@ -1647,7 +2072,7 @@
 		// on the command line in system encoding, and shows
 		// the file in console encoding
 		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"list Test1/" + systemDirName + "\" quit";
+			"-Wwarning \"list Test1/" + systemDirName + "\" quit";
 		queryout = LocalProcessStream(command.c_str(), 
 			bbackupquery_pid);
 		TEST_THAT(queryout.get() != NULL);
@@ -1671,31 +2096,34 @@
 		// Check that bbackupquery can compare the dir when given
 		// on the command line in system encoding.
 		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"compare -cEQ Test1/" + systemDirName +
+			"-Wwarning \"compare -cEQ Test1/" + systemDirName +
 			" testfiles/TestDir1/" + systemDirName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries:ReturnCode::Compare_Same);
 
 		// Check that bbackupquery can restore the dir when given
 		// on the command line in system encoding.
 		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"restore Test1/" + systemDirName +
+			"-Wwarning \"restore Test1/" + systemDirName +
 			" testfiles/restore-" + systemDirName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries:ReturnCode::Command_OK);
 
 		// Compare to make sure it was restored properly.
 		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"compare -cEQ Test1/" + systemDirName +
+			"-Wwarning \"compare -cEQ Test1/" + systemDirName +
 			" testfiles/restore-" + systemDirName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 
 		std::string fileToUnlink = "testfiles/restore-" + 
 			dirname + "/" + filename;
@@ -1704,22 +2132,25 @@
 		// Check that bbackupquery can get the file when given
 		// on the command line in system encoding.
 		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"get Test1/" + systemDirName + "/" + 
+			"-Wwarning \"get Test1/" + systemDirName + "/" + 
 			systemFileName + " " + "testfiles/restore-" + 
 			systemDirName + "/" + systemFileName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries:ReturnCode::Command_OK);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// And after changing directory to a relative path
-		command = BBACKUPQUERY " -c testfiles/bbackupd.conf -q "
+		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
+			"-Wwarning "
 			"\"lcd testfiles\" "
 			"\"cd Test1/" + systemDirName + "\" " + 
 			"\"get " + systemFileName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries:ReturnCode::Command_OK);
 		TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks");
 
 		// cannot overwrite a file that exists, so delete it
@@ -1727,34 +2158,38 @@
 		TEST_THAT(::unlink(tmp.c_str()) == 0);
 
 		// And after changing directory to an absolute path
-		command = BBACKUPQUERY " -c testfiles/bbackupd.conf -q "
+		command = BBACKUPQUERY " -c testfiles/bbackupd.conf -Wwarning "
 			"\"lcd " + cwd + "/testfiles\" "
 			"\"cd Test1/" + systemDirName + "\" " + 
 			"\"get " + systemFileName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
-		TEST_RETURN(compareReturnValue, 0);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries:ReturnCode::Command_OK);
 		TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks");
 
 		// Compare to make sure it was restored properly.
 		// The Get command does not restore attributes, so
 		// we must compare without them (-A) to succeed.
-		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"compare -cAEQ Test1/" + systemDirName +
+		command = BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-Wwarning \"compare -cAEQ Test1/" + systemDirName +
 			" testfiles/restore-" + systemDirName + "\" quit";
 
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries:ReturnCode::Compare_Same);
 
 		// Compare without attributes. This should fail.
-		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
-			"-q \"compare -cEQ Test1/" + systemDirName +
+		command = BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-Werror \"compare -cEQ Test1/" + systemDirName +
 			" testfiles/restore-" + systemDirName + "\" quit";
-
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
-		TEST_RETURN(compareReturnValue, 2);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
 #endif // WIN32
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -1831,11 +2266,13 @@
 			long start_time = time(NULL);
 
 			// check that no backup has run (compare fails)
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			int compareReturnValue = ::system(BBACKUPQUERY " "
+				"-Werror "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query3.log "
 				"\"compare -acQ\" quit");
-			TEST_RETURN(compareReturnValue, 2);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Different);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			TEST_THAT(unlink(sync_control_file) == 0);
@@ -1851,11 +2288,13 @@
 
 			wait_for_sync_end();
 			// check that backup has run (compare succeeds)
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
+				"-Wwarning "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query3a.log "
 				"\"compare -acQ\" quit");
-			TEST_RETURN(compareReturnValue, 1);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			if (failures > 0)
@@ -1900,19 +2339,21 @@
 	
 		// wait for backup daemon to do it's stuff, and compare again
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		int compareReturnValue = ::system(BBACKUPQUERY " -Wwarning "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query2.log "
 			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// Try a quick compare, just for fun
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query2q.log "
-			"\"compare -acqQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acqQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -1922,12 +2363,15 @@
 
 		// Check that store errors are reported neatly
 		printf("\n==== Create store error\n");
+
+		// break the store
 		TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf",
 			"testfiles/0_0/backup/01234567/info.rf.bak") == 0);
 		TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf",
 			"testfiles/0_1/backup/01234567/info.rf.bak") == 0);
 		TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf",
 			"testfiles/0_2/backup/01234567/info.rf.bak") == 0);
+
 		// Create a file to trigger an upload
 		{
 			int fd1 = open("testfiles/TestDir1/force-upload", 
@@ -1935,50 +2379,189 @@
 			TEST_THAT(fd1 > 0);
 			TEST_THAT(write(fd1, "just do it", 10) == 10);
 			TEST_THAT(close(fd1) == 0);
-			wait_for_backup_operation(4);
 		}
-		// Wait and test...
-		wait_for_backup_operation();
-		// Check that it was reported correctly
+
+		wait_for_backup_operation(4);
+		// Check that an error was reported just once
 		TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.1"));
-		// Check that the error was only reported once
 		TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.2"));
-		// Fix the store
+		// Now kill bbackupd and start one that's running in
+		// snapshot mode, check that it automatically syncs after
+		// an error, without waiting for another sync command.
+		terminate_bbackupd(bbackupd_pid);
+		std::string cmd = BBACKUPD " " + bbackupd_args + 
+			" testfiles/bbackupd-snapshot.conf";
+		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
+		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
+		::safe_sleep(1);
+		TEST_THAT(ServerIsAlive(bbackupd_pid));
+		TEST_THAT(ServerIsAlive(bbstored_pid));
+		if (!ServerIsAlive(bbackupd_pid)) return 1;
+		if (!ServerIsAlive(bbstored_pid)) return 1;
+
+		sync_and_wait();
+
+		// Check that the error was reported once more
+		TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.1"));
+		TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.2"));
+		TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.3"));
+		// Fix the store (so that bbackupquery compare works)
 		TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf.bak",
 			"testfiles/0_0/backup/01234567/info.rf") == 0);
 		TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf.bak",
 			"testfiles/0_1/backup/01234567/info.rf") == 0);
 		TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf.bak",
 			"testfiles/0_2/backup/01234567/info.rf") == 0);
+
+		// Check that we DO get errors on compare (cannot do this
+		// until after we fix the store, which creates a race)
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query3b.log "
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
+
+		// Test initial state
+		TEST_THAT(!TestFileExists("testfiles/"
+			"notifyran.backup-start.wait-snapshot.1"));
+
+		// Set a tag for the notify script to distinguist from
+		// previous runs.
+		{
+			int fd1 = open("testfiles/notifyscript.tag", 
+				O_CREAT | O_EXCL | O_WRONLY, 0700);
+			TEST_THAT(fd1 > 0);
+			TEST_THAT(write(fd1, "wait-snapshot", 13) == 13);
+			TEST_THAT(close(fd1) == 0);
+		}
+
+		// bbackupd should pause for about 90 seconds
+		wait_for_backup_operation(85);
+		TEST_THAT(!TestFileExists("testfiles/"
+			"notifyran.backup-start.wait-snapshot.1"));
+
+		// Should not have backed up, should still get errors
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query3b.log "
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
+
+		// wait another 10 seconds, bbackup should have run
+		wait_for_backup_operation(10);
+		TEST_THAT(TestFileExists("testfiles/"
+			"notifyran.backup-start.wait-snapshot.1"));
 	
-		// Check that we DO get errors on compare
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		// Check that it did get uploaded, and we have no more errors
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3b.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 2);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
 
+		TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0);
+
+		// Stop the snapshot bbackupd
+		terminate_bbackupd(bbackupd_pid);
+
+		// Break the store again
+		TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf",
+			"testfiles/0_0/backup/01234567/info.rf.bak") == 0);
+		TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf",
+			"testfiles/0_1/backup/01234567/info.rf.bak") == 0);
+		TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf",
+			"testfiles/0_2/backup/01234567/info.rf.bak") == 0);
+
+		// Modify a file to trigger an upload
+		{
+			int fd1 = open("testfiles/TestDir1/force-upload", 
+				O_WRONLY, 0700);
+			TEST_THAT(fd1 > 0);
+			TEST_THAT(write(fd1, "and again", 9) == 9);
+			TEST_THAT(close(fd1) == 0);
+		}
+
+		// Restart the old bbackupd, in automatic mode
+		cmd = BBACKUPD " " + bbackupd_args + 
+			" testfiles/bbackupd.conf";
+		bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid");
+		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
+		::safe_sleep(1);
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
 		TEST_THAT(ServerIsAlive(bbstored_pid));
 		if (!ServerIsAlive(bbackupd_pid)) return 1;
 		if (!ServerIsAlive(bbstored_pid)) return 1;
 
-		// Wait until bbackupd recovers from the exception
-		wait_for_backup_operation(100);
-
-		// Ensure that the force-upload file gets uploaded,
-		// meaning that bbackupd recovered
 		sync_and_wait();
 
+		// Fix the store again
+		TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf.bak",
+			"testfiles/0_0/backup/01234567/info.rf") == 0);
+		TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf.bak",
+			"testfiles/0_1/backup/01234567/info.rf") == 0);
+		TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf.bak",
+			"testfiles/0_2/backup/01234567/info.rf") == 0);
+
+		// Check that we DO get errors on compare (cannot do this
+		// until after we fix the store, which creates a race)
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query3b.log "
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
+
+		// Test initial state
+		TEST_THAT(!TestFileExists("testfiles/"
+			"notifyran.backup-start.wait-automatic.1"));
+
+		// Set a tag for the notify script to distinguist from
+		// previous runs.
+		{
+			int fd1 = open("testfiles/notifyscript.tag", 
+				O_CREAT | O_EXCL | O_WRONLY, 0700);
+			TEST_THAT(fd1 > 0);
+			TEST_THAT(write(fd1, "wait-automatic", 14) == 14);
+			TEST_THAT(close(fd1) == 0);
+		}
+
+		// bbackupd should pause for at least 90 seconds
+		wait_for_backup_operation(85);
+		TEST_THAT(!TestFileExists("testfiles/"
+			"notifyran.backup-start.wait-automatic.1"));
+
+		// Should not have backed up, should still get errors
+		compareReturnValue = ::system(BBACKUPQUERY " "
+			"-c testfiles/bbackupd.conf "
+			"-l testfiles/query3b.log "
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
+
+		// wait another 10 seconds, bbackup should have run
+		wait_for_backup_operation(10);
+		TEST_THAT(TestFileExists("testfiles/"
+			"notifyran.backup-start.wait-automatic.1"));
+	
 		// Check that it did get uploaded, and we have no more errors
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3b.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
 
+		TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0);
+
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
 		TEST_THAT(ServerIsAlive(bbstored_pid));
 		if (!ServerIsAlive(bbackupd_pid)) return 1;
@@ -2008,11 +2591,12 @@
 		#endif
 
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3c.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2036,11 +2620,12 @@
 		#endif
 
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3d.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2067,11 +2652,12 @@
 		#endif
 
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3e.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2098,11 +2684,12 @@
 		#endif
 
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3f.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2129,11 +2716,12 @@
 		wait_for_operation(5);
 		// back up both files
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3g.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		#ifdef WIN32
@@ -2146,11 +2734,12 @@
 		TEST_THAT(!TestFileExists("testfiles/TestDir1/untracked-1"));
 		TEST_THAT( TestFileExists("testfiles/TestDir1/untracked-2"));
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3g.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2180,11 +2769,12 @@
 		wait_for_operation(5);
 		// back up both files
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3h.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		#ifdef WIN32
@@ -2197,11 +2787,12 @@
 		TEST_THAT(!TestFileExists("testfiles/TestDir1/tracked-1"));
 		TEST_THAT( TestFileExists("testfiles/TestDir1/tracked-2"));
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3i.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 	
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2217,11 +2808,12 @@
 			"testfiles/TestDir1/x1/dsfdsfs98.fd") == 0);
 		
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3j.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2251,11 +2843,12 @@
 		
 		// Wait and test
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3k.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2309,11 +2902,12 @@
 		wait_for_sync_end(); // files too new
 		wait_for_sync_end(); // should (not) be backed up this time
 
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3l.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2337,19 +2931,21 @@
 		wait_for_sync_end();
 		
 		// compare with exclusions, should not find differences
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3m.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// compare without exclusions, should find differences
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3n.log "
-			"\"compare -acEQ\" quit");
-		TEST_RETURN(compareReturnValue, 2);
+			"-Werror \"compare -acEQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2421,13 +3017,14 @@
 
 			// Wait and test...
 			wait_for_backup_operation();
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query3o.log "
-				"\"compare -acQ\" quit");
+				"-Werror \"compare -acQ\" quit");
 
 			// should find differences
-			TEST_RETURN(compareReturnValue, 3);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Error);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// Check that it was reported correctly
@@ -2524,11 +3121,12 @@
 		
 		// Wait and test
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query4.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 	
 		printf("\n==== Restore files and directories\n");
@@ -2573,26 +3171,33 @@
 				== Restore_Complete);
 
 			// Make sure you can't restore to a nonexistant path
-			printf("\n\n==== Try to restore to a path "
+			printf("\n==== Try to restore to a path "
 				"that doesn't exist\n");
 			fflush(stdout);
-			TEST_THAT(BackupClientRestore(*client, restoredirid, 
-				"testfiles/no-such-path/subdir", 
-				true /* print progress dots */) 
-				== Restore_TargetPathNotFound);
 
+			{
+				Logging::Guard guard(Log::FATAL);
+				TEST_THAT(BackupClientRestore(*client,
+					restoredirid, 
+					"testfiles/no-such-path/subdir", 
+					true /* print progress dots */) 
+					== Restore_TargetPathNotFound);
+			}
+
 			// Log out
 			client->QueryFinished();
 			sSocket.Close();
 		}
 
 		// Compare the restored files
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query10.log "
+			"-Wwarning "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2606,12 +3211,14 @@
 			"testfiles\\restore-Test1\\f1.dat");
 		TEST_RETURN(compareReturnValue, 0);
 
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query10a.log "
+			"-Werror "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 2);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 	
 		// set it back, expect no failures
@@ -2619,11 +3226,13 @@
 			"testfiles\\restore-Test1\\f1.dat");
 		TEST_RETURN(compareReturnValue, 0);
 
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf -l testfiles/query10a.log "
+			"-Wwarning "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// change the timestamp on a file, expect a compare failure
@@ -2643,46 +3252,54 @@
 		// a compare failure
 		TEST_THAT(set_file_time(testfile, dummyTime, lastModTime,
 			lastAccessTime));
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query10a.log "
+			"-Werror "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 2);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// last access time is not backed up, so it cannot be compared
 		TEST_THAT(set_file_time(testfile, creationTime, lastModTime,
 			dummyTime));
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query10a.log "
+			"-Wwarning "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// last write time is backed up, so changing it should cause
 		// a compare failure
 		TEST_THAT(set_file_time(testfile, creationTime, dummyTime,
 			lastAccessTime));
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query10a.log "
+			"-Werror "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 2);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// set back to original values, check that compare succeeds
 		TEST_THAT(set_file_time(testfile, creationTime, lastModTime,
 			lastAccessTime));
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query10a.log "
+			"-Wwarning "
 			"\"compare -cEQ Test1 testfiles/restore-Test1\" "
 			"quit");
-		TEST_RETURN(compareReturnValue, 1);
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 #endif // WIN32
 
@@ -2705,11 +3322,12 @@
 		
 		// Wait and test
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query5.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2722,19 +3340,21 @@
 		TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", 
 			"testfiles/TestDir1/renamed-dir") == 0);
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query6.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// and again, but with quick flag
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query6q.log "
-			"\"compare -acqQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acqQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// Rename some files -- one under the threshold, others above
@@ -2746,11 +3366,12 @@
 		TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", 
 			"testfiles/TestDir1/find2perl-ren") == 0);
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query6.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2788,11 +3409,12 @@
 
 		// Wait and test
 		wait_for_backup_operation();
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query3e.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 1);
+			"-Wwarning \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Same);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2856,11 +3478,12 @@
 		// Wait and test that there *are* differences
 		wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION * 
 			3) / 2); // little bit longer than usual
-		compareReturnValue = ::system(BBACKUPQUERY " -q "
+		compareReturnValue = ::system(BBACKUPQUERY " "
 			"-c testfiles/bbackupd.conf "
 			"-l testfiles/query6.log "
-			"\"compare -acQ\" quit");
-		TEST_RETURN(compareReturnValue, 2);
+			"-Werror \"compare -acQ\" quit");
+		TEST_RETURN(compareReturnValue,
+			BackupQueries::ReturnCode::Compare_Different);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 	
 		TEST_THAT(ServerIsAlive(bbackupd_pid));
@@ -2914,12 +3537,13 @@
 			sSocket.Close();
 
 			// Then check it has restored the correct stuff
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query14.log "
-				"\"compare -cEQ Test1 "
+				"-Wwarning \"compare -cEQ Test1 "
 				"testfiles/restore-interrupt\" quit");
-			TEST_RETURN(compareReturnValue, 1);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		}
 #endif // !WIN32
@@ -2947,12 +3571,14 @@
 			sSocket.Close();
 
 			// Do a compare with the now undeleted files
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query11.log "
+				"-Wwarning "
 				"\"compare -cEQ Test1/x1 "
 				"testfiles/restore-Test1-x1-2\" quit");
-			TEST_RETURN(compareReturnValue, 1);
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		}
 		
@@ -3002,11 +3628,12 @@
 
 			// compare, and check that it works
 			// reports the correct error message (and finishes)
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query15a.log "
-				"\"compare -acQ\" quit");
-			TEST_RETURN(compareReturnValue, 1);
+				"-Wwarning \"compare -acQ\" quit");
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// open the file again, compare and check that compare
@@ -3015,22 +3642,24 @@
 				O_LOCK, 0);
 			TEST_THAT(handle != INVALID_HANDLE_VALUE);
 
-			compareReturnValue = ::system(BBACKUPQUERY 
-				" -q -c testfiles/bbackupd.conf "
+			compareReturnValue = ::system(BBACKUPQUERY " "
+				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query15.log "
-				"\"compare -acQ\" quit");
-			TEST_RETURN(compareReturnValue, 3);
+				"-Werror \"compare -acQ\" quit");
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Error);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// close the file again, check that compare
 			// works again
 			CloseHandle(handle);
 
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query15a.log "
-				"\"compare -acQ\" quit");
-			TEST_RETURN(compareReturnValue, 1);
+				"-Wwarning \"compare -acQ\" quit");
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		}
 #endif
@@ -3055,11 +3684,12 @@
 			// Wait and compare (a little bit longer than usual)
 			wait_for_backup_operation(
 				(TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); 
-			compareReturnValue = ::system(BBACKUPQUERY " -q "
+			compareReturnValue = ::system(BBACKUPQUERY " "
 				"-c testfiles/bbackupd.conf "
 				"-l testfiles/query4a.log "
-				"\"compare -acQ\" quit");
-			TEST_RETURN(compareReturnValue, 1);
+				"-Wwarning \"compare -acQ\" quit");
+			TEST_RETURN(compareReturnValue,
+				BackupQueries::ReturnCode::Compare_Same);
 			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 			// Kill it again
@@ -3067,10 +3697,12 @@
 		}
 	}
 
-	// List the files on the server
+	/*
+	// List the files on the server - why?
 	::system(BBACKUPQUERY " -q -c testfiles/bbackupd.conf "
 		"-l testfiles/queryLIST.log \"list -rotdh\" quit");
 	TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+	*/
 
 	#ifndef WIN32	
 		if(::getuid() == 0)

Modified: box/trunk/test/bbackupd/testfiles/extcheck1.pl.in
===================================================================
--- box/trunk/test/bbackupd/testfiles/extcheck1.pl.in	2008-05-28 15:28:36 UTC (rev 2183)
+++ box/trunk/test/bbackupd/testfiles/extcheck1.pl.in	2008-05-28 15:35:42 UTC (rev 2184)
@@ -3,7 +3,10 @@
 
 my $flags = $ARGV[0] or "";
 
-unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac$flags\" quit 2>&1 |")
+unless(open IN,"../../bin/bbackupquery/bbackupquery -Wwarning " .
+	"-c testfiles/bbackupd.conf " .
+	"-l testfiles/query4.log " .
+	"\"compare -ac$flags\" quit 2>&1 |")
 {
 	print "Couldn't open compare utility\n";
 	exit 2;

Modified: box/trunk/test/bbackupd/testfiles/extcheck2.pl.in
===================================================================
--- box/trunk/test/bbackupd/testfiles/extcheck2.pl.in	2008-05-28 15:28:36 UTC (rev 2183)
+++ box/trunk/test/bbackupd/testfiles/extcheck2.pl.in	2008-05-28 15:35:42 UTC (rev 2184)
@@ -3,7 +3,10 @@
 
 my $flags = $ARGV[0] or "";
 
-unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac$flags\" quit 2>&1 |")
+unless(open IN,"../../bin/bbackupquery/bbackupquery -Wwarning " .
+	"-c testfiles/bbackupd.conf " .
+	"-l testfiles/query4.log " .
+	"\"compare -ac$flags\" quit 2>&1 |")
 {
 	print "Couldn't open compare utility\n";
 	exit 2;



From boxbackup-dev at fluffy.co.uk  Wed May 28 16:59:01 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 16:59:01 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2185 - box/trunk/lib/common
Message-ID: <20080528155901.4BDA4325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 16:59:01 +0100 (Wed, 28 May 2008)
New Revision: 2185

Modified:
   box/trunk/lib/common/Test.cpp
Log:
Include "Box.h" in the right place.


Modified: box/trunk/lib/common/Test.cpp
===================================================================
--- box/trunk/lib/common/Test.cpp	2008-05-28 15:35:42 UTC (rev 2184)
+++ box/trunk/lib/common/Test.cpp	2008-05-28 15:59:01 UTC (rev 2185)
@@ -7,6 +7,8 @@
 //
 // --------------------------------------------------------------------------
 
+#include "Box.h"
+
 #include <errno.h>
 #include <signal.h>
 #include <stdio.h>
@@ -19,7 +21,6 @@
 	#include <unistd.h>
 #endif
 
-#include "Box.h"
 #include "Test.h"
 
 bool TestFileExists(const char *Filename)



From boxbackup-dev at fluffy.co.uk  Wed May 28 16:59:38 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 16:59:38 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2186 - box/trunk/lib/common
Message-ID: <20080528155938.C71AB325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 16:59:38 +0100 (Wed, 28 May 2008)
New Revision: 2186

Modified:
   box/trunk/lib/common/BoxPlatform.h
Log:
Check that Box.h has been included early enough.


Modified: box/trunk/lib/common/BoxPlatform.h
===================================================================
--- box/trunk/lib/common/BoxPlatform.h	2008-05-28 15:59:01 UTC (rev 2185)
+++ box/trunk/lib/common/BoxPlatform.h	2008-05-28 15:59:38 UTC (rev 2186)
@@ -29,6 +29,9 @@
 #endif
 
 #ifdef WIN32
+	#ifdef __MSVCRT_VERSION__
+		#error Must include Box.h before sys/types.h
+	#endif
 	// need msvcrt version 6.1 or higher for _gmtime64()
 	// must define this before importing <sys/types.h>
 	#define __MSVCRT_VERSION__ 0x0601



From boxbackup-dev at fluffy.co.uk  Wed May 28 18:04:19 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 18:04:19 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2187 - box/trunk/lib/common
Message-ID: <20080528170419.9BFC7325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 18:04:18 +0100 (Wed, 28 May 2008)
New Revision: 2187

Modified:
   box/trunk/lib/common/Utils.cpp
Log:
Fix possible memory corruption while dumping stack trace due to 
mismatching new []/delete.


Modified: box/trunk/lib/common/Utils.cpp
===================================================================
--- box/trunk/lib/common/Utils.cpp	2008-05-28 15:59:38 UTC (rev 2186)
+++ box/trunk/lib/common/Utils.cpp	2008-05-28 17:04:18 UTC (rev 2187)
@@ -78,6 +78,9 @@
 
 	BOX_TRACE("Obtained " << size << " stack frames.");
 
+	size_t output_len = 256;
+	char*  output_buf = new char [output_len];
+
 	for(i = 0; i < size; i++)
 	{
 		// Demangling code copied from 
@@ -92,12 +95,12 @@
 		std::string mangled_func = mangled_frame.substr(start + 1,
 			end - start - 1);
 
-		size_t len = 256;
-		std::auto_ptr<char> output_buf(new char [len]);
 		int status;
 		
-		if (abi::__cxa_demangle(mangled_func.c_str(), output_buf.get(),
-			&len, &status) == NULL)
+		char* result = abi::__cxa_demangle(mangled_func.c_str(),
+			output_buf, &output_len, &status);
+
+		if (result == NULL)
 		{
 			if (status == 0)
 			{
@@ -112,6 +115,7 @@
 			}
 			else if (status == -2)
 			{
+				// Probably non-C++ name, don't demangle
 				/*
 				BOX_WARNING("Demangle failed with "
 					"with invalid name: " <<
@@ -133,16 +137,18 @@
 		}
 		else
 		{
+			output_buf = result;
 			output_frame = mangled_frame.substr(0, start + 1) +
 				// std::string(output_buf.get()) +
-				output_buf.get() +
-				mangled_frame.substr(end);
+				result + mangled_frame.substr(end);
 		}
 		#endif // HAVE_CXXABI_H
 
 		BOX_TRACE("Stack frame " << i << ": " << output_frame);
 	}
 
+	delete [] output_buf;
+
 #include "MemLeakFindOff.h"
 	free (strings);
 #include "MemLeakFindOn.h"



From boxbackup-dev at fluffy.co.uk  Wed May 28 18:22:51 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Wed, 28 May 2008 18:22:51 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2188 - box/trunk/test/bbackupd
Message-ID: <20080528172251.6B4A1325023@www.boxbackup.org>

Author: chris
Date: 2008-05-28 18:22:51 +0100 (Wed, 28 May 2008)
New Revision: 2188

Modified:
   box/trunk/test/bbackupd/testbbackupd.cpp
Log:
Fix compile error on Windows.


Modified: box/trunk/test/bbackupd/testbbackupd.cpp
===================================================================
--- box/trunk/test/bbackupd/testbbackupd.cpp	2008-05-28 17:04:18 UTC (rev 2187)
+++ box/trunk/test/bbackupd/testbbackupd.cpp	2008-05-28 17:22:51 UTC (rev 2188)
@@ -2102,7 +2102,7 @@
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		TEST_RETURN(compareReturnValue,
-			BackupQueries:ReturnCode::Compare_Same);
+			BackupQueries::ReturnCode::Compare_Same);
 
 		// Check that bbackupquery can restore the dir when given
 		// on the command line in system encoding.
@@ -2113,7 +2113,7 @@
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		TEST_RETURN(compareReturnValue,
-			BackupQueries:ReturnCode::Command_OK);
+			BackupQueries::ReturnCode::Command_OK);
 
 		// Compare to make sure it was restored properly.
 		command = BBACKUPQUERY " -c testfiles/bbackupd.conf "
@@ -2138,7 +2138,7 @@
 
 		compareReturnValue = ::system(command.c_str());
 		TEST_RETURN(compareReturnValue,
-			BackupQueries:ReturnCode::Command_OK);
+			BackupQueries::ReturnCode::Command_OK);
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 
 		// And after changing directory to a relative path
@@ -2150,7 +2150,7 @@
 
 		compareReturnValue = ::system(command.c_str());
 		TEST_RETURN(compareReturnValue,
-			BackupQueries:ReturnCode::Command_OK);
+			BackupQueries::ReturnCode::Command_OK);
 		TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks");
 
 		// cannot overwrite a file that exists, so delete it
@@ -2165,7 +2165,7 @@
 
 		compareReturnValue = ::system(command.c_str());
 		TEST_RETURN(compareReturnValue,
-			BackupQueries:ReturnCode::Command_OK);
+			BackupQueries::ReturnCode::Command_OK);
 		TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks");
 
 		// Compare to make sure it was restored properly.
@@ -2179,7 +2179,7 @@
 		compareReturnValue = ::system(command.c_str());
 		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
 		TEST_RETURN(compareReturnValue,
-			BackupQueries:ReturnCode::Compare_Same);
+			BackupQueries::ReturnCode::Compare_Same);
 
 		// Compare without attributes. This should fail.
 		command = BBACKUPQUERY " "



From boxbackup-dev at fluffy.co.uk  Thu May 29 22:51:01 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Thu, 29 May 2008 22:51:01 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2189 - box/trunk/contrib/rpm
Message-ID: <20080529215102.00BB2325019@www.boxbackup.org>

Author: martin
Date: 2008-05-29 22:51:00 +0100 (Thu, 29 May 2008)
New Revision: 2189

Modified:
   box/trunk/contrib/rpm/boxbackup.spec
Log:
Fix paths to bbreporter files

Modified: box/trunk/contrib/rpm/boxbackup.spec
===================================================================
--- box/trunk/contrib/rpm/boxbackup.spec	2008-05-28 17:22:51 UTC (rev 2188)
+++ box/trunk/contrib/rpm/boxbackup.spec	2008-05-29 21:51:00 UTC (rev 2189)
@@ -39,7 +39,7 @@
 Release: 1
 License: BSD
 Group: Applications/Archiving
-Packager: Martin Ebourne <boxbackup-dev at boxbackup.org>
+Packager: boxbackup-dev at boxbackup.org
 URL: http://www.boxbackup.org/
 Source0: %{ident}.tgz
 Requires: openssl >= 0.9.7a
@@ -122,9 +122,9 @@
 install -m 644 %{distribution_dir}THANKS.txt \
 	$RPM_BUILD_ROOT%{_docdir}/%{ident}
 
-install -m 644 %{distribution_dir}contrib/bbreporter/LICENSE \
+install -m 644 contrib/bbreporter/LICENSE \
 	$RPM_BUILD_ROOT%{_docdir}/%{ident}/bbreporter
-install -m 755 %{distribution_dir}contrib/bbreporter/bbreporter.py \
+install -m 755 contrib/bbreporter/bbreporter.py \
 	$RPM_BUILD_ROOT%{_docdir}/%{ident}/bbreporter
 
 # Client
@@ -225,6 +225,9 @@
 %doc %{_docdir}/%{ident}/bbreporter
 
 %changelog
+* Thu May 29 2008 Martin Ebourne <martin at zepler.org>
+- Fix paths to bbreporter files
+
 * Sat Jan 13 2006 Chris Wilson <chris+box at qwirx.com>
 - Support building from an unofficial tarball (from svn) by changing
   %{distribution_dir} at the top.



From boxbackup-dev at fluffy.co.uk  Thu May 29 22:51:38 2008
From: boxbackup-dev at fluffy.co.uk (boxbackup-dev at fluffy.co.uk)
Date: Thu, 29 May 2008 22:51:38 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2190 - box/trunk/lib/common
Message-ID: <20080529215138.E147E325019@www.boxbackup.org>

Author: martin
Date: 2008-05-29 22:51:38 +0100 (Thu, 29 May 2008)
New Revision: 2190

Modified:
   box/trunk/lib/common/Logging.h
Log:
Fix strerror include. Needed for gcc 4.3

Modified: box/trunk/lib/common/Logging.h
===================================================================
--- box/trunk/lib/common/Logging.h	2008-05-29 21:51:00 UTC (rev 2189)
+++ box/trunk/lib/common/Logging.h	2008-05-29 21:51:38 UTC (rev 2190)
@@ -10,6 +10,7 @@
 #ifndef LOGGING__H
 #define LOGGING__H
 
+#include <cstring>
 #include <iomanip>
 #include <sstream>
 #include <vector>
@@ -49,11 +50,11 @@
 	{ BOX_LOG(Log::TRACE, stuff) }
 
 #define BOX_LOG_SYS_WARNING(stuff) \
-	BOX_WARNING(stuff << ": " << strerror(errno) << " (" << errno << ")")
+	BOX_WARNING(stuff << ": " << std::strerror(errno) << " (" << errno << ")")
 #define BOX_LOG_SYS_ERROR(stuff) \
-	BOX_ERROR(stuff << ": " << strerror(errno) << " (" << errno << ")")
+	BOX_ERROR(stuff << ": " << std::strerror(errno) << " (" << errno << ")")
 #define BOX_LOG_SYS_FATAL(stuff) \
-	BOX_FATAL(stuff << ": " << strerror(errno) << " (" << errno << ")")
+	BOX_FATAL(stuff << ": " << std::strerror(errno) << " (" << errno << ")")
 
 #ifdef WIN32
 	#define BOX_LOG_WIN_ERROR(stuff) \