[Box Backup-commit] COMMIT r2535 - in box/trunk: bin/bbstored test/backupstore

boxbackup-dev at boxbackup.org boxbackup-dev at boxbackup.org
Sun Jun 28 13:04:26 BST 2009


Author: chris
Date: 2009-06-28 13:04:25 +0100 (Sun, 28 Jun 2009)
New Revision: 2535

Modified:
   box/trunk/bin/bbstored/BackupStoreContext.cpp
   box/trunk/bin/bbstored/BackupStoreContext.h
   box/trunk/test/backupstore/testbackupstore.cpp
Log:
Update reference counts when files and directories are uploaded.


Modified: box/trunk/bin/bbstored/BackupStoreContext.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.cpp	2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/bin/bbstored/BackupStoreContext.cpp	2009-06-28 12:04:25 UTC (rev 2535)
@@ -191,6 +191,23 @@
 	
 	// Keep the pointer to it
 	mpStoreInfo = i;
+
+	BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+	// try to load the reference count database
+	try
+	{
+		mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+	}
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database is missing or corrupted, "
+			"creating a new one, expect housekeeping to find and "
+			"fix problems with reference counts later.");
+		
+		BackupStoreRefCountDatabase::CreateForRegeneration(account);
+		mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+	}
 }
 
 
@@ -395,15 +412,18 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-//		Purpose: Add a file to the store, from a given stream, into a specified directory.
-//				 Returns object ID of the new file.
+//		Name:    BackupStoreContext::AddFile(IOStream &, int64_t,
+//			 int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Add a file to the store, from a given stream, into
+//			 a specified directory. Returns object ID of the new
+//			 file.
 //		Created: 2003/09/03
 //
 // --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
-		int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
-		bool MarkFileWithSameNameAsOldVersions)
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+	int64_t ModificationTime, int64_t AttributesHash,
+	int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+	bool MarkFileWithSameNameAsOldVersions)
 {
 	if(mpStoreInfo.get() == 0)
 	{
@@ -670,6 +690,9 @@
 	mpStoreInfo->ChangeBlocksUsed(blocksUsed);
 	mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
 	
+	// Increment reference count on the new directory to one
+	mapRefCount->AddReference(id);
+	
 	// Save the store info -- can cope if this exceptions because infomation
 	// will be rebuilt by housekeeping, and ID allocation can recover.
 	SaveStoreInfo();
@@ -768,8 +791,9 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-//		Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+//		Name:    BackupStoreContext::UndeleteFile(int64_t, int64_t)
+//		Purpose: Undeletes a file, if it exists, returning true if
+//			 the file existed.
 //		Created: 2003/10/21
 //
 // --------------------------------------------------------------------------
@@ -933,8 +957,10 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
-//		Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+//		Name:    BackupStoreContext::AddDirectory(int64_t,
+//			 const BackupStoreFilename &, bool &)
+//		Purpose: Creates a directory (or just returns the ID of an
+//			 existing one). rAlreadyExists set appropraitely.
 //		Created: 2003/09/04
 //
 // --------------------------------------------------------------------------
@@ -974,7 +1000,7 @@
 	// Allocate the next ID
 	int64_t id = AllocateObjectID();
 
-	// Create a blank directory with the given attributes on disc
+	// Create an empty directory with the given attributes on disc
 	std::string fn;
 	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
 	{
@@ -998,11 +1024,14 @@
 		// Not added to cache, so don't set the size in the directory
 	}
 	
-	// Then add it into the directory
+	// Then add it into the parent directory
 	try
 	{
 		dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
 		SaveDirectory(dir, InDirectory);
+
+		// Increment reference count on the new directory to one
+		mapRefCount->AddReference(id);
 	}
 	catch(...)
 	{
@@ -1016,7 +1045,7 @@
 		// Don't worry about the incremented number in the store info
 		throw;	
 	}
-	
+
 	// Save the store info (may be postponed)
 	SaveStoreInfo();
 

Modified: box/trunk/bin/bbstored/BackupStoreContext.h
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.h	2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/bin/bbstored/BackupStoreContext.h	2009-06-28 12:04:25 UTC (rev 2535)
@@ -14,6 +14,7 @@
 #include <map>
 #include <memory>
 
+#include "BackupStoreRefCountDatabase.h"
 #include "NamedLock.h"
 #include "ProtocolObject.h"
 #include "Utils.h"
@@ -137,7 +138,6 @@
 	void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
 	int64_t AllocateObjectID();
 
-private:
 	int32_t mClientID;
 	HousekeepingInterface &mrDaemon;
 	int mProtocolPhase;
@@ -150,6 +150,9 @@
 	
 	// Store info
 	std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+
+	// Refcount database
+	std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
 	
 	// Directory cache
 	std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;

Modified: box/trunk/test/backupstore/testbackupstore.cpp
===================================================================
--- box/trunk/test/backupstore/testbackupstore.cpp	2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/test/backupstore/testbackupstore.cpp	2009-06-28 12:04:25 UTC (rev 2535)
@@ -34,6 +34,8 @@
 #include "BackupClientFileAttributes.h"
 #include "BackupClientCryptoKeys.h"
 #include "ServerControl.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
 
 #include "MemLeakFindOn.h"
 
@@ -470,8 +472,28 @@
 	TEST_THAT(dirs == 0 || dirs == 2);
 }
 
-int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth)
+void create_file_in_dir(std::string name, std::string source, int64_t parentId,
+	BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount)
 {
+	BackupStoreFilenameClear name_encoded("file_One");
+	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(
+		source.c_str(), parentId, name_encoded));
+	std::auto_ptr<BackupProtocolClientSuccess> stored(
+		protocol.QueryStoreFile(
+			parentId,
+			0x123456789abcdefLL,		/* modification time */
+			0x7362383249872dfLL,		/* attr hash */
+			0,				/* diff from ID */
+			name_encoded,
+			*upload));
+	int64_t objectId = stored->GetObjectID();
+	TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed());
+	TEST_EQUAL(1, rRefCount.GetRefCount(objectId))
+}
+
+int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
+	const char *name, int depth, BackupStoreRefCountDatabase& rRefCount)
+{
 	// Create a directory
 	int64_t subdirid = 0;
 	BackupStoreFilenameClear dirname(name);
@@ -486,49 +508,31 @@
 	}
 	
 	printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
+
+	std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+		BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
+	std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+		BackupStoreRefCountDatabase::Load(
+			apAccounts->GetEntry(0x1234567), true));
+	TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed());
+	TEST_EQUAL(1, rRefCount.GetRefCount(subdirid))
 	
 	// Put more directories in it, if we haven't gone down too far
 	if(depth > 0)
 	{
-		create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1);
-		create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1);
+		create_test_data_subdirs(protocol, subdirid, "dir_One",
+			depth - 1, rRefCount);
+		create_test_data_subdirs(protocol, subdirid, "dir_Two",
+			depth - 1, rRefCount);
 	}
 	
 	// Stick some files in it
-	{
-		BackupStoreFilenameClear name("file_One");
-		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
-		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
-			subdirid,
-			0x123456789abcdefLL,		/* modification time */
-			0x7362383249872dfLL,		/* attr hash */
-			0,							/* diff from ID */
-			name,
-			*upload));
-	}
-	{
-		BackupStoreFilenameClear name("file_Two");
-		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
-		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
-			subdirid,
-			0x123456789abcdefLL,		/* modification time */
-			0x7362383249872dfLL,		/* attr hash */
-			0,							/* diff from ID */
-			name,
-			*upload));
-	}
-	{
-		BackupStoreFilenameClear name("file_Three");
-		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
-		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
-			subdirid,
-			0x123456789abcdefLL,		/* modification time */
-			0x7362383249872dfLL,		/* attr hash */
-			0,							/* diff from ID */
-			name,
-			*upload));
-	}
-
+	create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol,
+		rRefCount);
+	create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol,
+		rRefCount);
+	create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol,
+		rRefCount);
 	return subdirid;
 }
 
@@ -885,16 +889,49 @@
 	}
 }
 
-
-int test_server(const char *hostname)
+void init_context(TLSContext& rContext)
 {
-	// Context
-	TLSContext context;
-	context.Initialise(false /* client */,
+	rContext.Initialise(false /* client */,
 			"testfiles/clientCerts.pem",
 			"testfiles/clientPrivKey.pem",
 			"testfiles/clientTrustedCAs.pem");
+}
 
+std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname,
+	TLSContext& rContext)
+{
+	init_context(rContext);
+	std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS);
+	conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+	return conn;
+}
+
+std::auto_ptr<BackupProtocolClient> test_server_login(SocketStreamTLS& rConn)
+{
+	// Make a protocol
+	std::auto_ptr<BackupProtocolClient> protocol(new
+		BackupProtocolClient(rConn));
+	
+	// Check the version
+	std::auto_ptr<BackupProtocolClientVersion> serverVersion(
+		protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION));
+	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+	// Login
+	std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(
+		protocol->QueryLogin(0x01234567, 0));
+
+	return protocol;
+}
+
+int test_server(const char *hostname)
+{
+	TLSContext context;
+	std::auto_ptr<SocketStreamTLS> conn = open_conn(hostname, context);
+	std::auto_ptr<BackupProtocolClient> apProtocol(
+		test_server_login(*conn));
+	BackupProtocolClient& protocol(*apProtocol);
+
 	// Make some test attributes
 	#define ATTR1_SIZE 	245
 	#define ATTR2_SIZE 	23
@@ -911,28 +948,11 @@
 
 	// BLOCK
 	{
-		// Open a connection to the server
-		SocketStreamTLS conn;
-		conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
-
-		// Make a protocol
-		BackupProtocolClient protocol(conn);
-		
 		// Get it logging
 		FILE *protocolLog = ::fopen("testfiles/protocol.log", "w");
 		TEST_THAT(protocolLog != 0);
 		protocol.SetLogToFile(protocolLog);
 
-		// Check the version
-		std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
-		TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
-
-		// Login
-		std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
-		
-		// Check marker is 0
-		TEST_THAT(loginConf->GetClientStoreMarker() == 0);
-
 #ifndef WIN32
 		// Check that we can't open a new connection which requests write permissions
 		{
@@ -1435,10 +1455,18 @@
 		}
 		
 //}	skip:
+
+		std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+			BackupStoreAccountDatabase::Read(
+				"testfiles/accounts.txt"));
+		std::auto_ptr<BackupStoreRefCountDatabase> apRefCount(
+			BackupStoreRefCountDatabase::Load(
+				apAccounts->GetEntry(0x1234567), true));
 	
 		// Create some nice recursive directories
 		int64_t dirtodelete = create_test_data_subdirs(protocol,
-			BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */);
+			BackupProtocolClientListDirectory::RootDirectory,
+			"test_delete", 6 /* depth */, *apRefCount);
 		
 		// And delete them
 		{
@@ -1760,7 +1788,33 @@
 		TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567"));
 		TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8);
 		// make sure something is written to it
+		
+		std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+			BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
 
+		std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+			BackupStoreRefCountDatabase::Load(
+				apAccounts->GetEntry(0x1234567), true));
+		TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
+			apReferences->GetLastObjectIDUsed());
+		TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
+		apReferences.reset();
+
+		// Delete the refcount database and log in again,
+		// check that it is recreated automatically but with
+		// no objects in it, to ensure seamless upgrade.
+		TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
+
+		// Context
+		TLSContext context;
+		std::auto_ptr<SocketStreamTLS> conn = open_conn("localhost",
+			context);
+		test_server_login(*conn)->QueryFinished();
+
+		apReferences = BackupStoreRefCountDatabase::Load(
+			apAccounts->GetEntry(0x1234567), true);
+		TEST_EQUAL(0, apReferences->GetLastObjectIDUsed());
+
 		TEST_THAT(ServerIsAlive(pid));
 
 		TEST_THAT(test_server("localhost") == 0);




More information about the Boxbackup-commit mailing list