[Box Backup-commit] COMMIT r3580 - box/trunk/bin/bbackupd

subversion at boxbackup.org subversion at boxbackup.org
Wed Jun 10 20:46:20 BST 2015


Author: chris
Date: 2015-06-10 19:46:20 +0000 (Wed, 10 Jun 2015)
New Revision: 3580

Modified:
   box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
   box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp
   box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h
Log:
Switch inode database to storing values using Archive.

This allows us to store the local path to the tracked file in the database,
and will allow us to extend the storage format again in future if needed.

Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp	2015-06-10 19:46:17 UTC (rev 3579)
+++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp	2015-06-10 19:46:20 UTC (rev 3580)
@@ -207,8 +207,8 @@
 		{
 			BackupClientInodeToIDMap &idMap(
 				rParams.mrContext.GetNewIDMap());
-			idMap.AddToMap(dest_st.st_ino, mObjectID,
-				ContainingDirectoryID);
+			idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID,
+				ConvertVssPathToRealPath(rLocalPath, rBackupLocation));
 		}
 		// Add attributes to checksum
 		currentStateChecksum.Add(&dest_st.st_mode,
@@ -1227,18 +1227,8 @@
 			BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap());
 		
 			// Need to get an ID from somewhere...
-			if(latestObjectID != 0)
+			if(latestObjectID == 0)
 			{
-				// Use this one
-				BOX_TRACE("Storing uploaded file ID " <<
-					inodeNum << " (" << nonVssFilePath << ") "
-					"in ID map as object " <<
-					latestObjectID << " with parent " <<
-					mObjectID);
-				idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */);
-			}
-			else
-			{
 				// Don't know it -- haven't sent anything to the store, and didn't get a listing.
 				// Look it up in the current map, and if it's there, use that.
 				const BackupClientInodeToIDMap &currentIDMap(rContext.GetCurrentIDMap());
@@ -1263,13 +1253,23 @@
 					// into it. However, in a long running process this may happen occasionally and
 					// not indicate anything wrong.
 					// Run the release version for real life use, where this check is not made.
-					BOX_TRACE("Storing found file ID " << inodeNum <<
-						" (" << nonVssFilePath << ") in ID map as "
-						"object " << objid << " with parent " << mObjectID);
-					idMap.AddToMap(inodeNum, objid,
-						mObjectID /* containing directory */);
+
+					latestObjectID = objid;
 				}
 			}
+
+			if(latestObjectID != 0)
+			{
+				BOX_TRACE("Storing uploaded file ID " <<
+					inodeNum << " (" << nonVssFilePath << ") "
+					"in ID map as object " <<
+					latestObjectID << " with parent " <<
+					mObjectID);
+				idMap.AddToMap(inodeNum, latestObjectID,
+					mObjectID /* containing directory */,
+					nonVssFilePath);
+			}
+
 		}
 
 		if (fileSynced)

Modified: box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp	2015-06-10 19:46:17 UTC (rev 3579)
+++ box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp	2015-06-10 19:46:20 UTC (rev 3580)
@@ -16,15 +16,16 @@
 #include "BackupClientInodeToIDMap.h"
 #undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION
 
+#include "Archive.h"
 #include "BackupStoreException.h"
+#include "CollectInBufferStream.h"
+#include "MemBlockStream.h"
+#include "autogen_CommonException.h"
 
 #include "MemLeakFindOn.h"
 
-typedef struct
-{
-	int64_t mObjectID;
-	int64_t mInDirectory;
-} IDBRecord;
+#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version"
+#define BOX_DBM_INODE_DB_VERSION_CURRENT 2
 
 #define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode)
 
@@ -118,12 +119,58 @@
 	
 	if(!mpDepot)
 	{
-		BOX_WARNING(BOX_DBM_MESSAGE("Failed to open inode "
-			"database: " << mFilename));
 		THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure,
 			BOX_DBM_MESSAGE("Failed to open inode database: " <<
 				mFilename));
 	}
+
+	const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY;
+	int32_t version = 0;
+
+	if(CreateNew)
+	{
+		version = BOX_DBM_INODE_DB_VERSION_CURRENT;
+
+		int ret = dpput(mpDepot, version_key, strlen(version_key),
+			(char *)(&version), sizeof(version), DP_DKEEP);
+
+		if(!ret)
+		{
+			THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure,
+				BOX_DBM_MESSAGE("Failed to write version number to inode "
+					"database: " << mFilename));
+		}
+	}
+	else
+	{
+		int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0,
+			sizeof(version), (char *)(&version));
+
+		if(ret == -1)
+		{
+			THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure,
+				"Missing version number in inode database. Perhaps it "
+				"needs to be recreated: " << mFilename);
+		}
+
+		if(ret != sizeof(version))
+		{
+			THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure,
+				"Wrong size version number in inode database: expected "
+				<< sizeof(version) << " bytes but found " << ret);
+		}
+
+		if(version != BOX_DBM_INODE_DB_VERSION_CURRENT)
+		{
+			THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure,
+				"Wrong version number in inode database: expected " <<
+				BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " <<
+				version << ". Perhaps it needs to be recreated: " <<
+				mFilename);
+		}
+
+		// By this point the version number has been checked and is OK.
+	}
 	
 	// Read only flag
 	mReadOnly = ReadOnly;
@@ -175,7 +222,7 @@
 //
 // --------------------------------------------------------------------------
 void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID,
-	int64_t InDirectory)
+	int64_t InDirectory, const std::string& LocalPath)
 {
 	if(mReadOnly)
 	{
@@ -190,12 +237,15 @@
 	ASSERT_DBM_OPEN();
 
 	// Setup structures
-	IDBRecord rec;
-	rec.mObjectID = ObjectID;
-	rec.mInDirectory = InDirectory;
+	CollectInBufferStream buf;
+	Archive arc(buf, IOStream::TimeOutInfinite);
+	arc.WriteExact((uint64_t)ObjectID);
+	arc.WriteExact((uint64_t)InDirectory);
+	arc.Write(LocalPath);
+	buf.SetForReading();
 
 	ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef),
-		(const char *)&rec, sizeof(rec), DP_DOVER),
+		(const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER),
 		"Failed to add record to inode database", mFilename,
 		BackupStoreException, BerkelyDBFailure);
 }
@@ -211,8 +261,8 @@
 //		Created: 11/11/03
 //
 // --------------------------------------------------------------------------
-bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef,
-	int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const
+bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut,
+	int64_t &rInDirectoryOut, std::string* pLocalPathOut) const
 {
 	if(mEmpty)
 	{
@@ -226,20 +276,45 @@
 	}
 	
 	ASSERT_DBM_OPEN();
-
-	IDBRecord rec;
-	
-	if(dpgetwb(mpDepot, (const char *)&InodeRef, sizeof(InodeRef),
-		0, sizeof(IDBRecord), (char *)&rec) == -1)
+	int size;
+	char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef),
+		0, -1, &size);
+	if(data == NULL)
 	{
 		// key not in file
 		return false;
 	}
-		
+
+	// Free data automatically when the guard goes out of scope.
+	MemoryBlockGuard<char *> guard(data);
+	MemBlockStream stream(data, size);
+	Archive arc(stream, IOStream::TimeOutInfinite);
+
 	// Return data
-	rObjectIDOut = rec.mObjectID;
-	rInDirectoryOut = rec.mInDirectory;
+	try
+	{
+		arc.Read(rObjectIDOut);
+		arc.Read(rInDirectoryOut);
+		if(pLocalPathOut)
+		{
+			arc.Read(*pLocalPathOut);
+		}
+	}
+	catch(CommonException &e)
+	{
+		if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead)
+		{
+			THROW_FILE_ERROR("Failed to lookup record in inode database: "
+				<< InodeRef << ": not enough data in record", mFilename,
+				BackupStoreException, BerkelyDBFailure);
+			// Need to throw precisely that exception to ensure that the
+			// invalid database is deleted, so that we don't hit the same
+			// error next time.
+		}
 
+		throw;
+	}
+
 	// Found
 	return true;
 }

Modified: box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h
===================================================================
--- box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h	2015-06-10 19:46:17 UTC (rev 3579)
+++ box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h	2015-06-10 19:46:20 UTC (rev 3580)
@@ -40,8 +40,10 @@
 	void Open(const char *Filename, bool ReadOnly, bool CreateNew);
 	void OpenEmpty();
 
-	void AddToMap(InodeRefType InodeRef, int64_t ObjectID, int64_t InDirectory);
-	bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const;
+	void AddToMap(InodeRefType InodeRef, int64_t ObjectID,
+		int64_t InDirectory, const std::string& LocalPath);
+	bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut,
+		int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const;
 
 	void Close();
 




More information about the Boxbackup-commit mailing list