[Box Backup-commit] COMMIT r2945 - in box/trunk: . bin/bbstored lib/backupclient lib/backupstore test/bbackupd
subversion at boxbackup.org
subversion at boxbackup.org
Tue Apr 26 19:44:27 BST 2011
Author: chris
Date: 2011-04-26 19:44:26 +0100 (Tue, 26 Apr 2011)
New Revision: 2945
Added:
box/trunk/lib/backupstore/BackupClientFileAttributes.cpp
box/trunk/lib/backupstore/BackupClientFileAttributes.h
box/trunk/lib/backupstore/BackupCommands.cpp
box/trunk/lib/backupstore/BackupConstants.h
box/trunk/lib/backupstore/BackupStoreConstants.h
box/trunk/lib/backupstore/BackupStoreContext.cpp
box/trunk/lib/backupstore/BackupStoreContext.h
box/trunk/lib/backupstore/BackupStoreDirectory.cpp
box/trunk/lib/backupstore/BackupStoreDirectory.h
box/trunk/lib/backupstore/BackupStoreException.h
box/trunk/lib/backupstore/BackupStoreException.txt
box/trunk/lib/backupstore/BackupStoreFile.cpp
box/trunk/lib/backupstore/BackupStoreFile.h
box/trunk/lib/backupstore/BackupStoreFileCryptVar.cpp
box/trunk/lib/backupstore/BackupStoreFileCryptVar.h
box/trunk/lib/backupstore/BackupStoreFileEncodeStream.cpp
box/trunk/lib/backupstore/BackupStoreFileEncodeStream.h
box/trunk/lib/backupstore/BackupStoreFileRevDiff.cpp
box/trunk/lib/backupstore/BackupStoreFileWire.h
box/trunk/lib/backupstore/BackupStoreFilename.cpp
box/trunk/lib/backupstore/BackupStoreFilename.h
box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp
box/trunk/lib/backupstore/BackupStoreFilenameClear.h
box/trunk/lib/backupstore/BackupStoreObjectMagic.h
box/trunk/lib/backupstore/Makefile.extra
box/trunk/lib/backupstore/RunStatusProvider.h
box/trunk/lib/backupstore/backupprotocol.txt
Removed:
box/trunk/bin/bbstored/BackupCommands.cpp
box/trunk/bin/bbstored/BackupConstants.h
box/trunk/bin/bbstored/BackupStoreContext.cpp
box/trunk/bin/bbstored/BackupStoreContext.h
box/trunk/bin/bbstored/Makefile.extra
box/trunk/bin/bbstored/backupprotocol.txt
box/trunk/lib/backupclient/BackupClientFileAttributes.cpp
box/trunk/lib/backupclient/BackupClientFileAttributes.h
box/trunk/lib/backupclient/BackupStoreConstants.h
box/trunk/lib/backupclient/BackupStoreDirectory.cpp
box/trunk/lib/backupclient/BackupStoreDirectory.h
box/trunk/lib/backupclient/BackupStoreException.h
box/trunk/lib/backupclient/BackupStoreException.txt
box/trunk/lib/backupclient/BackupStoreFile.cpp
box/trunk/lib/backupclient/BackupStoreFile.h
box/trunk/lib/backupclient/BackupStoreFileCryptVar.cpp
box/trunk/lib/backupclient/BackupStoreFileCryptVar.h
box/trunk/lib/backupclient/BackupStoreFileEncodeStream.cpp
box/trunk/lib/backupclient/BackupStoreFileEncodeStream.h
box/trunk/lib/backupclient/BackupStoreFileRevDiff.cpp
box/trunk/lib/backupclient/BackupStoreFileWire.h
box/trunk/lib/backupclient/BackupStoreFilename.cpp
box/trunk/lib/backupclient/BackupStoreFilename.h
box/trunk/lib/backupclient/BackupStoreFilenameClear.cpp
box/trunk/lib/backupclient/BackupStoreFilenameClear.h
box/trunk/lib/backupclient/BackupStoreObjectMagic.h
box/trunk/lib/backupclient/Makefile.extra
box/trunk/lib/backupclient/RunStatusProvider.h
Modified:
box/trunk/modules.txt
box/trunk/test/bbackupd/Makefile.extra
Log:
Major refactoring to make lib/backupclient depend on lib/backupstore rather
than the other way around. This is needed to allow clients to have all the
code that they'd need to implement local backups (using the Local protocol)
in subsequent commits.
Deleted: box/trunk/bin/bbstored/BackupCommands.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupCommands.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/bin/bbstored/BackupCommands.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,959 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupCommands.cpp
-// Purpose: Implement commands for the Backup store protocol
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include <set>
-#include <sstream>
-
-#include "autogen_BackupProtocolServer.h"
-#include "autogen_RaidFileException.h"
-#include "BackupConstants.h"
-#include "BackupStoreContext.h"
-#include "BackupStoreConstants.h"
-#include "BackupStoreDirectory.h"
-#include "BackupStoreException.h"
-#include "BackupStoreFile.h"
-#include "BackupStoreInfo.h"
-#include "BufferedStream.h"
-#include "CollectInBufferStream.h"
-#include "FileStream.h"
-#include "InvisibleTempFileStream.h"
-#include "RaidFileController.h"
-#include "StreamableMemBlock.h"
-
-#include "MemLeakFindOn.h"
-
-#define PROTOCOL_ERROR(code) \
- std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( \
- BackupProtocolServerError::ErrorType, \
- BackupProtocolServerError::code));
-
-#define CHECK_PHASE(phase) \
- if(rContext.GetPhase() != BackupStoreContext::phase) \
- { \
- return PROTOCOL_ERROR(Err_NotInRightProtocolPhase); \
- }
-
-#define CHECK_WRITEABLE_SESSION \
- if(rContext.SessionIsReadOnly()) \
- { \
- return PROTOCOL_ERROR(Err_SessionReadOnly); \
- }
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Return the current version, or an error if the requested version isn't allowed
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Version)
-
- // Correct version?
- if(mVersion != BACKUP_STORE_SERVER_VERSION)
- {
- return PROTOCOL_ERROR(Err_WrongVersion);
- }
-
- // Mark the next phase
- rContext.SetPhase(BackupStoreContext::Phase_Login);
-
- // Return our version
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Return the current version, or an error if the requested version isn't allowed
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Login)
-
- // Check given client ID against the ID in the certificate certificate
- // and that the client actually has an account on this machine
- if(mClientID != rContext.GetClientID())
- {
- BOX_WARNING("Failed login from client ID " <<
- BOX_FORMAT_ACCOUNT(mClientID) <<
- ": wrong certificate for this account");
- return PROTOCOL_ERROR(Err_BadLogin);
- }
-
- if(!rContext.GetClientHasAccount())
- {
- BOX_WARNING("Failed login from client ID " <<
- BOX_FORMAT_ACCOUNT(mClientID) <<
- ": no such account on this server");
- return PROTOCOL_ERROR(Err_BadLogin);
- }
-
- // If we need to write, check that nothing else has got a write lock
- if((mFlags & Flags_ReadOnly) != Flags_ReadOnly)
- {
- // See if the context will get the lock
- if(!rContext.AttemptToGetWriteLock())
- {
- BOX_WARNING("Failed to get write lock for Client ID " <<
- BOX_FORMAT_ACCOUNT(mClientID));
- return PROTOCOL_ERROR(Err_CannotLockStoreForWriting);
- }
-
- // Debug: check we got the lock
- ASSERT(!rContext.SessionIsReadOnly());
- }
-
- // Load the store info
- rContext.LoadStoreInfo();
-
- // Get the last client store marker
- int64_t clientStoreMarker = rContext.GetClientStoreMarker();
-
- // Mark the next phase
- rContext.SetPhase(BackupStoreContext::Phase_Commands);
-
- // Log login
- BOX_NOTICE("Login from Client ID " <<
- BOX_FORMAT_ACCOUNT(mClientID) <<
- " " <<
- (((mFlags & Flags_ReadOnly) != Flags_ReadOnly)
- ?"Read/Write":"Read-only"));
-
- // Get the usage info for reporting to the client
- int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0;
- rContext.GetStoreDiscUsageInfo(blocksUsed, blocksSoftLimit, blocksHardLimit);
-
- // Return success
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerLoginConfirmed(clientStoreMarker, blocksUsed, blocksSoftLimit, blocksHardLimit));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Marks end of conversation (Protocol framework handles this)
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- BOX_NOTICE("Session finished for Client ID " <<
- BOX_FORMAT_ACCOUNT(rContext.GetClientID()));
-
- // Let the context know about it
- rContext.ReceivedFinishCommand();
-
- // can be called in any phase
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerFinished);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Command to list a directory
-// Created: 2003/09/02
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Store the listing to a stream
- std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
-
- try
- {
- // Ask the context for a directory
- const BackupStoreDirectory &rdir(
- rContext.GetDirectory(mObjectID));
- rdir.WriteToStream(*stream, mFlagsMustBeSet,
- mFlagsNotToBeSet, mSendAttributes,
- false /* never send dependency info to the client */);
- }
- catch (RaidFileException &e)
- {
- if (e.GetSubType() == RaidFileException::RaidFileDoesntExist)
- {
- return PROTOCOL_ERROR(Err_DoesNotExist);
- }
- throw;
- }
-
- stream->SetForReading();
-
- // Get the protocol to send the stream
- rProtocol.SendStreamAfterCommand(stream.release());
-
- return std::auto_ptr<ProtocolObject>(
- new BackupProtocolServerSuccess(mObjectID));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Command to store a file on the server
-// Created: 2003/09/02
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- std::auto_ptr<ProtocolObject> hookResult =
- rContext.StartCommandHook(*this);
- if(hookResult.get())
- {
- return hookResult;
- }
-
- // Check that the diff from file actually exists, if it's specified
- if(mDiffFromFileID != 0)
- {
- if(!rContext.ObjectExists(mDiffFromFileID,
- BackupStoreContext::ObjectExists_File))
- {
- return PROTOCOL_ERROR(Err_DiffFromFileDoesNotExist);
- }
- }
-
- // A stream follows, which contains the file
- std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream());
-
- // Ask the context to store it
- int64_t id = 0;
- try
- {
- id = rContext.AddFile(*dirstream, mDirectoryObjectID,
- mModificationTime, mAttributesHash, mDiffFromFileID,
- mFilename,
- true /* mark files with same name as old versions */);
- }
- catch(BackupStoreException &e)
- {
- if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify)
- {
- return PROTOCOL_ERROR(Err_FileDoesNotVerify);
- }
- else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit)
- {
- return PROTOCOL_ERROR(Err_StorageLimitExceeded);
- }
- else
- {
- throw;
- }
- }
-
- // Tell the caller what the file was
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
-}
-
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Command to get an arbitary object from the server
-// Created: 2003/09/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Check the object exists
- if(!rContext.ObjectExists(mObjectID))
- {
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(NoObject));
- }
-
- // Open the object
- std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
-
- // Stream it to the peer
- rProtocol.SendStreamAfterCommand(object.release());
-
- // Tell the caller what the file was
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Command to get an file object from the server -- may have to do a bit of
-// work to get the object.
-// Created: 2003/09/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Check the objects exist
- if(!rContext.ObjectExists(mObjectID)
- || !rContext.ObjectExists(mInDirectory))
- {
- return PROTOCOL_ERROR(Err_DoesNotExist);
- }
-
- // Get the directory it's in
- const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory));
-
- // Find the object within the directory
- BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID);
- if(pfileEntry == 0)
- {
- return PROTOCOL_ERROR(Err_DoesNotExistInDirectory);
- }
-
- // The result
- std::auto_ptr<IOStream> stream;
-
- // Does this depend on anything?
- if(pfileEntry->GetDependsNewer() != 0)
- {
- // File exists, but is a patch from a new version. Generate the older version.
- std::vector<int64_t> patchChain;
- int64_t id = mObjectID;
- BackupStoreDirectory::Entry *en = 0;
- do
- {
- patchChain.push_back(id);
- en = rdir.FindEntryByID(id);
- if(en == 0)
- {
- BOX_ERROR("Object " <<
- BOX_FORMAT_OBJECTID(mObjectID) <<
- " in dir " <<
- BOX_FORMAT_OBJECTID(mInDirectory) <<
- " for account " <<
- BOX_FORMAT_ACCOUNT(rContext.GetClientID()) <<
- " references object " <<
- BOX_FORMAT_OBJECTID(id) <<
- " which does not exist in dir");
- return PROTOCOL_ERROR(Err_PatchConsistencyError);
- }
- id = en->GetDependsNewer();
- }
- while(en != 0 && id != 0);
-
- // OK! The last entry in the chain is the full file, the others are patches back from it.
- // Open the last one, which is the current from file
- std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1]));
-
- // Then, for each patch in the chain, do a combine
- for(int p = ((int)patchChain.size()) - 2; p >= 0; --p)
- {
- // ID of patch
- int64_t patchID = patchChain[p];
-
- // Open it a couple of times
- std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID));
- std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID));
-
- // Choose a temporary filename for the result of the combination
- std::ostringstream fs;
- fs << rContext.GetStoreRoot() << ".recombinetemp." << p;
- std::string tempFn =
- RaidFileController::DiscSetPathToFileSystemPath(
- rContext.GetStoreDiscSet(), fs.str(),
- p + 16);
-
- // Open the temporary file
- std::auto_ptr<IOStream> combined;
- try
- {
- {
- // Write nastily to allow this to work with gcc 2.x
- std::auto_ptr<IOStream> t(
- new InvisibleTempFileStream(
- tempFn.c_str(),
- O_RDWR | O_CREAT |
- O_EXCL | O_BINARY |
- O_TRUNC));
- combined = t;
- }
- }
- catch(...)
- {
- // Make sure it goes
- ::unlink(tempFn.c_str());
- throw;
- }
-
- // Do the combining
- BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined);
-
- // Move to the beginning of the combined file
- combined->Seek(0, IOStream::SeekType_Absolute);
-
- // Then shuffle round for the next go
- if (from.get()) from->Close();
- from = combined;
- }
-
- // Now, from contains a nice file to send to the client. Reorder it
- {
- // Write nastily to allow this to work with gcc 2.x
- std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */));
- stream = t;
- }
-
- // Release from file to avoid double deletion
- from.release();
- }
- else
- {
- // Simple case: file already exists on disc ready to go
-
- // Open the object
- std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
- BufferedStream buf(*object);
-
- // Verify it
- if(!BackupStoreFile::VerifyEncodedFileFormat(buf))
- {
- return PROTOCOL_ERROR(Err_FileDoesNotVerify);
- }
-
- // Reset stream -- seek to beginning
- object->Seek(0, IOStream::SeekType_Absolute);
-
- // Reorder the stream/file into stream order
- {
- // Write nastily to allow this to work with gcc 2.x
- std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */));
- stream = t;
- }
-
- // Object will be deleted when the stream is deleted,
- // so can release the object auto_ptr here to avoid
- // premature deletion
- object.release();
- }
-
- // Stream the reordered stream to the peer
- rProtocol.SendStreamAfterCommand(stream.get());
-
- // Don't delete the stream here
- stream.release();
-
- // Tell the caller what the file was
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Create directory command
-// Created: 2003/09/04
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Get the stream containing the attributes
- std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
- // Collect the attributes -- do this now so no matter what the outcome,
- // the data has been absorbed.
- StreamableMemBlock attr;
- attr.Set(*attrstream, rProtocol.GetTimeout());
-
- // Check to see if the hard limit has been exceeded
- if(rContext.HardLimitExceeded())
- {
- // Won't allow creation if the limit has been exceeded
- return PROTOCOL_ERROR(Err_StorageLimitExceeded);
- }
-
- bool alreadyExists = false;
- int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists);
-
- if(alreadyExists)
- {
- return PROTOCOL_ERROR(Err_DirectoryAlreadyExists);
- }
-
- // Tell the caller what the file was
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Change attributes on directory
-// Created: 2003/09/06
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Get the stream containing the attributes
- std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
- // Collect the attributes -- do this now so no matter what the outcome,
- // the data has been absorbed.
- StreamableMemBlock attr;
- attr.Set(*attrstream, rProtocol.GetTimeout());
-
- // Get the context to do it's magic
- rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime);
-
- // Tell the caller what the file was
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupStoreContext &)
-// Purpose: Change attributes on directory
-// Created: 2003/09/06
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Get the stream containing the attributes
- std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
- // Collect the attributes -- do this now so no matter what the outcome,
- // the data has been absorbed.
- StreamableMemBlock attr;
- attr.Set(*attrstream, rProtocol.GetTimeout());
-
- // Get the context to do it's magic
- int64_t objectID = 0;
- if(!rContext.ChangeFileAttributes(mFilename, mInDirectory, attr, mAttributesHash, objectID))
- {
- // Didn't exist
- return PROTOCOL_ERROR(Err_DoesNotExist);
- }
-
- // Tell the caller what the file was
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Delete a file
-// Created: 2003/10/21
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Context handles this
- int64_t objectID = 0;
- rContext.DeleteFile(mFilename, mInDirectory, objectID);
-
- // return the object ID or zero for not found
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerUndeleteFile::DoCommand(
-// BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Undelete a file
-// Created: 2008-09-12
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteFile::DoCommand(
- BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Context handles this
- bool result = rContext.UndeleteFile(mObjectID, mInDirectory);
-
- // return the object ID or zero for not found
- return std::auto_ptr<ProtocolObject>(
- new BackupProtocolServerSuccess(result ? mObjectID : 0));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Delete a directory
-// Created: 2003/10/21
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Check it's not asking for the root directory to be deleted
- if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
- {
- return PROTOCOL_ERROR(Err_CannotDeleteRoot);
- }
-
- // Context handles this
- try
- {
- rContext.DeleteDirectory(mObjectID);
- }
- catch (BackupStoreException &e)
- {
- if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject)
- {
- return PROTOCOL_ERROR(Err_MultiplyReferencedObject);
- }
-
- throw;
- }
-
- // return the object ID
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Undelete a directory
-// Created: 23/11/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Check it's not asking for the root directory to be deleted
- if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
- {
- return PROTOCOL_ERROR(Err_CannotDeleteRoot);
- }
-
- // Context handles this
- rContext.DeleteDirectory(mObjectID, true /* undelete */);
-
- // return the object ID
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Command to set the client's store marker
-// Created: 2003/10/29
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Set the marker
- rContext.SetClientStoreMarker(mClientStoreMarker);
-
- // return store marker set
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mClientStoreMarker));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Command to move an object from one directory to another
-// Created: 2003/11/12
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
- CHECK_WRITEABLE_SESSION
-
- // Let context do this, but modify error reporting on exceptions...
- try
- {
- rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory,
- mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName,
- (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject);
- }
- catch(BackupStoreException &e)
- {
- if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory)
- {
- return PROTOCOL_ERROR(Err_DoesNotExist);
- }
- else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory)
- {
- return PROTOCOL_ERROR(Err_TargetNameExists);
- }
- else
- {
- throw;
- }
- }
-
- // Return the object ID
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Command to find the name of an object
-// Created: 12/11/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Create a stream for the list of filenames
- std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
-
- // Object and directory IDs
- int64_t objectID = mObjectID;
- int64_t dirID = mContainingDirectoryID;
-
- // Data to return in the reply
- int32_t numNameElements = 0;
- int16_t objectFlags = 0;
- int64_t modTime = 0;
- uint64_t attrModHash = 0;
- bool haveModTimes = false;
-
- do
- {
- // Check the directory really exists
- if(!rContext.ObjectExists(dirID, BackupStoreContext::ObjectExists_Directory))
- {
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
- }
-
- // Load up the directory
- const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID));
-
- // Find the element in this directory and store it's name
- if(objectID != ObjectID_DirectoryOnly)
- {
- const BackupStoreDirectory::Entry *en = rdir.FindEntryByID(objectID);
-
- // If this can't be found, then there is a problem... tell the caller it can't be found
- if(en == 0)
- {
- // Abort!
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
- }
-
- // Store flags?
- if(objectFlags == 0)
- {
- objectFlags = en->GetFlags();
- }
-
- // Store modification times?
- if(!haveModTimes)
- {
- modTime = en->GetModificationTime();
- attrModHash = en->GetAttributesHash();
- haveModTimes = true;
- }
-
- // Store the name in the stream
- en->GetName().WriteToStream(*stream);
-
- // Count of name elements
- ++numNameElements;
- }
-
- // Setup for next time round
- objectID = dirID;
- dirID = rdir.GetContainerID();
-
- } while(objectID != 0 && objectID != BACKUPSTORE_ROOT_DIRECTORY_ID);
-
- // Stream to send?
- if(numNameElements > 0)
- {
- // Get the stream ready to go
- stream->SetForReading();
- // Tell the protocol to send the stream
- rProtocol.SendStreamAfterCommand(stream.release());
- }
-
- // Make reply
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(numNameElements, modTime, attrModHash, objectFlags));
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Get the block index from a file, by ID
-// Created: 19/1/04
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Open the file
- std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID));
-
- // Move the file pointer to the block index
- BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
-
- // Return the stream to the client
- rProtocol.SendStreamAfterCommand(stream.release());
-
- // Return the object ID
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Get the block index from a file, by name within a directory
-// Created: 19/1/04
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Get the directory
- const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory));
-
- // Find the latest object ID within it which has the same name
- int64_t objectID = 0;
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *en = 0;
- while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
- {
- if(en->GetName() == mFilename)
- {
- // Store the ID, if it's a newer ID than the last one
- if(en->GetObjectID() > objectID)
- {
- objectID = en->GetObjectID();
- }
- }
- }
-
- // Found anything?
- if(objectID == 0)
- {
- // No... return a zero object ID
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(0));
- }
-
- // Open the file
- std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID));
-
- // Move the file pointer to the block index
- BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
-
- // Return the stream to the client
- rProtocol.SendStreamAfterCommand(stream.release());
-
- // Return the object ID
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Return the amount of disc space used
-// Created: 19/4/04
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- // Get store info from context
- const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo());
-
- // Find block size
- RaidFileController &rcontroller(RaidFileController::GetController());
- RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber()));
-
- // Return info
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerAccountUsage(
- rinfo.GetBlocksUsed(),
- rinfo.GetBlocksInOldFiles(),
- rinfo.GetBlocksInDeletedFiles(),
- rinfo.GetBlocksInDirectories(),
- rinfo.GetBlocksSoftLimit(),
- rinfo.GetBlocksHardLimit(),
- rdiscSet.GetBlockSize()
- ));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupStoreContext &)
-// Purpose: Return the amount of disc space used
-// Created: 19/4/04
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<ProtocolObject> BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
-{
- CHECK_PHASE(Phase_Commands)
-
- //
- // NOOP
- //
- return std::auto_ptr<ProtocolObject>(new BackupProtocolServerIsAlive());
-}
Deleted: box/trunk/bin/bbstored/BackupConstants.h
===================================================================
--- box/trunk/bin/bbstored/BackupConstants.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/bin/bbstored/BackupConstants.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,21 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupConstants.h
-// Purpose: Constants for the backup server and client
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPCONSTANTS__H
-#define BACKUPCONSTANTS__H
-
-// 15 minutes to timeout (milliseconds)
-#define BACKUP_STORE_TIMEOUT (15*60*1000)
-
-// Should the store daemon convert files to Raid immediately?
-#define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true
-
-#endif // BACKUPCONSTANTS__H
-
-
Deleted: box/trunk/bin/bbstored/BackupStoreContext.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/bin/bbstored/BackupStoreContext.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,1808 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreContext.cpp
-// Purpose: Context for backup store server
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include <stdio.h>
-
-#include "BackupConstants.h"
-#include "BackupStoreContext.h"
-#include "BackupStoreDaemon.h"
-#include "BackupStoreDirectory.h"
-#include "BackupStoreException.h"
-#include "BackupStoreFile.h"
-#include "BackupStoreInfo.h"
-#include "BackupStoreObjectMagic.h"
-#include "BufferedStream.h"
-#include "BufferedWriteStream.h"
-#include "FileStream.h"
-#include "InvisibleTempFileStream.h"
-#include "RaidFileController.h"
-#include "RaidFileRead.h"
-#include "RaidFileWrite.h"
-#include "StoreStructure.h"
-
-#include "MemLeakFindOn.h"
-
-
-// Maximum number of directories to keep in the cache
-// When the cache is bigger than this, everything gets
-// deleted.
-#ifdef BOX_RELEASE_BUILD
- #define MAX_CACHE_SIZE 32
-#else
- #define MAX_CACHE_SIZE 2
-#endif
-
-// Allow the housekeeping process 4 seconds to release an account
-#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4
-
-// Maximum amount of store info updates before it's actually saved to disc.
-#define STORE_INFO_SAVE_DELAY 96
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::BackupStoreContext()
-// Purpose: Constructor
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-BackupStoreContext::BackupStoreContext(int32_t ClientID,
- HousekeepingInterface &rDaemon)
- : mClientID(ClientID),
- mrDaemon(rDaemon),
- mProtocolPhase(Phase_START),
- mClientHasAccount(false),
- mStoreDiscSet(-1),
- mReadOnly(true),
- mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY),
- mpTestHook(NULL)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::~BackupStoreContext()
-// Purpose: Destructor
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-BackupStoreContext::~BackupStoreContext()
-{
- // Delete the objects in the cache
- for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
- {
- delete (i->second);
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::CleanUp()
-// Purpose: Clean up after a connection
-// Created: 16/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::CleanUp()
-{
- // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified
- if(mapStoreInfo.get() && !(mapStoreInfo->IsReadOnly()) &&
- mapStoreInfo->IsModified())
- {
- mapStoreInfo->Save();
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::ReceivedFinishCommand()
-// Purpose: Called when the finish command is received by the protocol
-// Created: 16/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::ReceivedFinishCommand()
-{
- if(!mReadOnly && mapStoreInfo.get())
- {
- // Save the store info, not delayed
- SaveStoreInfo(false);
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::AttemptToGetWriteLock()
-// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags
-// Created: 2003/09/02
-//
-// --------------------------------------------------------------------------
-bool BackupStoreContext::AttemptToGetWriteLock()
-{
- // Make the filename of the write lock file
- std::string writeLockFile;
- StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile);
-
- // Request the lock
- bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
-
- if(!gotLock)
- {
- // The housekeeping process might have the thing open -- ask it to stop
- char msg[256];
- int msgLen = sprintf(msg, "r%x\n", mClientID);
- // Send message
- mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen);
-
- // Then try again a few times
- int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT;
- do
- {
- ::sleep(1 /* second */);
- --tries;
- gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
-
- } while(!gotLock && tries > 0);
- }
-
- if(gotLock)
- {
- // Got the lock, mark as not read only
- mReadOnly = false;
- }
-
- return gotLock;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::LoadStoreInfo()
-// Purpose: Load the store info from disc
-// Created: 2003/09/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::LoadStoreInfo()
-{
- if(mapStoreInfo.get() != 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded)
- }
-
- // Load it up!
- std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly));
-
- // Check it
- if(i->GetAccountID() != mClientID)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount)
- }
-
- // Keep the pointer to it
- mapStoreInfo = i;
-
- BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
-
- // try to load the reference count database
- try
- {
- mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
- }
- catch(BoxException &e)
- {
- BOX_WARNING("Reference count database is missing or corrupted, "
- "creating a new one, expect housekeeping to find and "
- "fix problems with reference counts later.");
-
- BackupStoreRefCountDatabase::CreateForRegeneration(account);
- mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::SaveStoreInfo(bool)
-// Purpose: Potentially delayed saving of the store info
-// Created: 16/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::SaveStoreInfo(bool AllowDelay)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // Can delay saving it a little while?
- if(AllowDelay)
- {
- --mSaveStoreInfoDelay;
- if(mSaveStoreInfoDelay > 0)
- {
- return;
- }
- }
-
- // Want to save now
- mapStoreInfo->Save();
-
- // Set count for next delay
- mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool)
-// Purpose: Create the filename of an object in the store, optionally creating the
-// containing directory if it doesn't already exist.
-// Created: 2003/09/02
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists)
-{
- // Delegate to utility function
- StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::GetDirectoryInternal(int64_t)
-// Purpose: Return a reference to a directory. Valid only until the
-// next time a function which affects directories is called.
-// Mainly this funciton, and creation of files.
-// Private version of this, which returns non-const directories.
-// Created: 2003/09/02
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID)
-{
- // Get the filename
- std::string filename;
- MakeObjectFilename(ObjectID, filename);
-
- // Already in cache?
- std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
- if(item != mDirectoryCache.end())
- {
- // Check the revision ID of the file -- does it need refreshing?
- int64_t revID = 0;
- if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID))
- {
- THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted)
- }
-
- if(revID == item->second->GetRevisionID())
- {
- // Looks good... return the cached object
- BOX_TRACE("Returning object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " from cache, modtime = " << revID);
- return *(item->second);
- }
-
- BOX_TRACE("Refreshing object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " in cache, modtime changed from " <<
- item->second->GetRevisionID() << " to " << revID);
-
- // Delete this cached object
- delete item->second;
- mDirectoryCache.erase(item);
- }
-
- // Need to load it up
-
- // First check to see if the cache is too big
- if(mDirectoryCache.size() > MAX_CACHE_SIZE)
- {
- // Very simple. Just delete everything!
- for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
- {
- delete (i->second);
- }
- mDirectoryCache.clear();
- }
-
- // Get a RaidFileRead to read it
- int64_t revID = 0;
- std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID));
- ASSERT(revID != 0);
-
- // New directory object
- std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory);
-
- // Read it from the stream, then set it's revision ID
- BufferedStream buf(*objectFile);
- dir->ReadFromStream(buf, IOStream::TimeOutInfinite);
- dir->SetRevisionID(revID);
-
- // Make sure the size of the directory is available for writing the dir back
- int64_t dirSize = objectFile->GetDiscUsageInBlocks();
- ASSERT(dirSize > 0);
- dir->SetUserInfo1_SizeInBlocks(dirSize);
-
- // Store in cache
- BackupStoreDirectory *pdir = dir.release();
- try
- {
- mDirectoryCache[ObjectID] = pdir;
- }
- catch(...)
- {
- delete pdir;
- throw;
- }
-
- // Return it
- return *pdir;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::AllocateObjectID()
-// Purpose: Allocate a new object ID, tolerant of failures to save store info
-// Created: 16/12/03
-//
-// --------------------------------------------------------------------------
-int64_t BackupStoreContext::AllocateObjectID()
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY
- // times after it has been updated, this is a reasonable number of times
- // to try for finding an unused ID.
- // (Sizes used in the store info are fixed by the housekeeping process)
- int retryLimit = (STORE_INFO_SAVE_DELAY * 2);
-
- while(retryLimit > 0)
- {
- // Attempt to allocate an ID from the store
- int64_t id = mapStoreInfo->AllocateObjectID();
-
- // Generate filename
- std::string filename;
- MakeObjectFilename(id, filename);
- // Check it doesn't exist
- if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
- {
- // Success!
- return id;
- }
-
- // Decrement retry count, and try again
- --retryLimit;
-
- // Mark that the store info should be saved as soon as possible
- mSaveStoreInfoDelay = 0;
-
- BOX_WARNING("When allocating object ID, found that " <<
- BOX_FORMAT_OBJECTID(id) << " is already in use");
- }
-
- THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation)
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::AddFile(IOStream &, int64_t,
-// int64_t, int64_t, const BackupStoreFilename &, bool)
-// Purpose: Add a file to the store, from a given stream, into
-// a specified directory. Returns object ID of the new
-// file.
-// Created: 2003/09/03
-//
-// --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
- int64_t ModificationTime, int64_t AttributesHash,
- int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
- bool MarkFileWithSameNameAsOldVersions)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // This is going to be a bit complex to make sure it copes OK
- // with things going wrong.
- // The only thing which isn't safe is incrementing the object ID
- // and keeping the blocks used entirely accurate -- but these
- // aren't big problems if they go horribly wrong. The sizes will
- // be corrected the next time the account has a housekeeping run,
- // and the object ID allocation code is tolerant of missed IDs.
- // (the info is written lazily, so these are necessary)
-
- // Get the directory we want to modify
- BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
-
- // Allocate the next ID
- int64_t id = AllocateObjectID();
-
- // Stream the file to disc
- std::string fn;
- MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
- int64_t newObjectBlocksUsed = 0;
- RaidFileWrite *ppreviousVerStoreFile = 0;
- bool reversedDiffIsCompletelyDifferent = false;
- int64_t oldVersionNewBlocksUsed = 0;
- try
- {
- RaidFileWrite storeFile(mStoreDiscSet, fn);
- storeFile.Open(false /* no overwriting */);
-
- // size adjustment from use of patch in old file
- int64_t spaceSavedByConversionToPatch = 0;
-
- // Diff or full file?
- if(DiffFromFileID == 0)
- {
- // A full file, just store to disc
- if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT))
- {
- THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
- }
- }
- else
- {
- // Check that the diffed from ID actually exists in the directory
- if(dir.FindEntryByID(DiffFromFileID) == 0)
- {
- THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory)
- }
-
- // Diff file, needs to be recreated.
- // Choose a temporary filename.
- std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp",
- 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */));
-
- try
- {
- // Open it twice
-#ifdef WIN32
- InvisibleTempFileStream diff(tempFn.c_str(),
- O_RDWR | O_CREAT | O_BINARY);
- InvisibleTempFileStream diff2(tempFn.c_str(),
- O_RDWR | O_BINARY);
-#else
- FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL);
- FileStream diff2(tempFn.c_str(), O_RDONLY);
-
- // Unlink it immediately, so it definitely goes away
- if(::unlink(tempFn.c_str()) != 0)
- {
- THROW_EXCEPTION(CommonException, OSFileError);
- }
-#endif
-
- // Stream the incoming diff to this temporary file
- if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT))
- {
- THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
- }
-
- // Verify the diff
- diff.Seek(0, IOStream::SeekType_Absolute);
- if(!BackupStoreFile::VerifyEncodedFileFormat(diff))
- {
- THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
- }
-
- // Seek to beginning of diff file
- diff.Seek(0, IOStream::SeekType_Absolute);
-
- // Filename of the old version
- std::string oldVersionFilename;
- MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */);
-
- // Reassemble that diff -- open previous file, and combine the patch and file
- std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
- BackupStoreFile::CombineFile(diff, diff2, *from, storeFile);
-
- // Then... reverse the patch back (open the from file again, and create a write file to overwrite it)
- std::auto_ptr<RaidFileRead> from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
- ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename);
- ppreviousVerStoreFile->Open(true /* allow overwriting */);
- from->Seek(0, IOStream::SeekType_Absolute);
- diff.Seek(0, IOStream::SeekType_Absolute);
- BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile,
- DiffFromFileID, &reversedDiffIsCompletelyDifferent);
-
- // Store disc space used
- oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks();
-
- // And make a space adjustment for the size calculation
- spaceSavedByConversionToPatch =
- from->GetDiscUsageInBlocks() -
- oldVersionNewBlocksUsed;
-
- // Everything cleans up here...
- }
- catch(...)
- {
- // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway
- ::unlink(tempFn.c_str());
- throw;
- }
- }
-
- // Get the blocks used
- newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks();
-
- // Exceeds the hard limit?
- int64_t newBlocksUsed = mapStoreInfo->GetBlocksUsed() +
- newObjectBlocksUsed - spaceSavedByConversionToPatch;
- if(newBlocksUsed > mapStoreInfo->GetBlocksHardLimit())
- {
- THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit)
- // The store file will be deleted automatically by the RaidFile object
- }
-
- // Commit the file
- storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
- }
- catch(...)
- {
- // Delete any previous version store file
- if(ppreviousVerStoreFile != 0)
- {
- delete ppreviousVerStoreFile;
- ppreviousVerStoreFile = 0;
- }
-
- throw;
- }
-
- // Verify the file -- only necessary for non-diffed versions
- // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because
- // in the non-diffed code path it's never allocated.
- if(DiffFromFileID == 0)
- {
- std::auto_ptr<RaidFileRead> checkFile(RaidFileRead::Open(mStoreDiscSet, fn));
- if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile))
- {
- // Error! Delete the file
- RaidFileWrite del(mStoreDiscSet, fn);
- del.Delete();
-
- // Exception
- THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
- }
- }
-
- // Modify the directory -- first make all files with the same name
- // marked as an old version
- int64_t blocksInOldFiles = 0;
- try
- {
- if(MarkFileWithSameNameAsOldVersions)
- {
- BackupStoreDirectory::Iterator i(dir);
-
- BackupStoreDirectory::Entry *e = 0;
- while((e = i.Next()) != 0)
- {
- // First, check it's not an old version (cheaper comparison)
- if(! e->IsOld())
- {
- // Compare name
- if(e->GetName() == rFilename)
- {
- // Check that it's definately not an old version
- ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0);
- // Set old version flag
- e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion);
- // Can safely do this, because we know we won't be here if it's already
- // an old version
- blocksInOldFiles += e->GetSizeInBlocks();
- }
- }
- }
- }
-
- // Then the new entry
- BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename,
- ModificationTime, id, newObjectBlocksUsed,
- BackupStoreDirectory::Entry::Flags_File,
- AttributesHash);
-
- // Adjust for the patch back stuff?
- if(DiffFromFileID != 0)
- {
- // Get old version entry
- BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID);
- ASSERT(poldEntry != 0);
-
- // Adjust dependency info of file?
- if(!reversedDiffIsCompletelyDifferent)
- {
- poldEntry->SetDependsNewer(id);
- pnewEntry->SetDependsOlder(DiffFromFileID);
- }
-
- // Adjust size of old entry
- int64_t oldSize = poldEntry->GetSizeInBlocks();
- poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed);
-
- // And adjust blocks used count, for later adjustment
- newObjectBlocksUsed += (oldVersionNewBlocksUsed - oldSize);
- blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize);
- }
-
- // Write the directory back to disc
- SaveDirectory(dir, InDirectory);
-
- // Commit the old version's new patched version, now that the directory safely reflects
- // the state of the files on disc.
- if(ppreviousVerStoreFile != 0)
- {
- ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
- delete ppreviousVerStoreFile;
- ppreviousVerStoreFile = 0;
- }
- }
- catch(...)
- {
- // Back out on adding that file
- RaidFileWrite del(mStoreDiscSet, fn);
- del.Delete();
-
- // Remove this entry from the cache
- RemoveDirectoryFromCache(InDirectory);
-
- // Delete any previous version store file
- if(ppreviousVerStoreFile != 0)
- {
- delete ppreviousVerStoreFile;
- ppreviousVerStoreFile = 0;
- }
-
- // Don't worry about the incremented number in the store info
- throw;
- }
-
- // Check logic
- ASSERT(ppreviousVerStoreFile == 0);
-
- // Modify the store info
-
- if(DiffFromFileID == 0)
- {
- mapStoreInfo->AdjustNumFiles(1);
- }
- else
- {
- mapStoreInfo->AdjustNumOldFiles(1);
- }
-
- mapStoreInfo->ChangeBlocksUsed(newObjectBlocksUsed);
- mapStoreInfo->ChangeBlocksInCurrentFiles(newObjectBlocksUsed -
- blocksInOldFiles);
- mapStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
-
- // Increment reference count on the new directory to one
- mapRefCount->AddReference(id);
-
- // Save the store info -- can cope if this exceptions because infomation
- // will be rebuilt by housekeeping, and ID allocation can recover.
- SaveStoreInfo(false);
-
- // Return the ID to the caller
- return id;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
-// Created: 2003/10/21
-//
-// --------------------------------------------------------------------------
-bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut)
-{
- // Essential checks!
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // Find the directory the file is in (will exception if it fails)
- BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
-
- // Setup flags
- bool fileExisted = false;
- bool madeChanges = false;
- rObjectIDOut = 0; // not found
-
- // Count of deleted blocks
- int64_t blocksDel = 0;
-
- try
- {
- // Iterate through directory, only looking at files which haven't been deleted
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *e = 0;
- while((e = i.Next(BackupStoreDirectory::Entry::Flags_File,
- BackupStoreDirectory::Entry::Flags_Deleted)) != 0)
- {
- // Compare name
- if(e->GetName() == rFilename)
- {
- // Check that it's definately not already deleted
- ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0);
- // Set deleted flag
- e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
- // Mark as made a change
- madeChanges = true;
- // Can safely do this, because we know we won't be here if it's already
- // an old version
- blocksDel += e->GetSizeInBlocks();
- // Is this the last version?
- if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
- {
- // Yes. It's been found.
- rObjectIDOut = e->GetObjectID();
- fileExisted = true;
- }
- }
- }
-
- // Save changes?
- if(madeChanges)
- {
- // Save the directory back
- SaveDirectory(dir, InDirectory);
-
- // Modify the store info, and write
- // It definitely wasn't an old or deleted version
- mapStoreInfo->AdjustNumFiles(-1);
- mapStoreInfo->AdjustNumDeletedFiles(1);
- mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel);
-
- SaveStoreInfo(false);
- }
- }
- catch(...)
- {
- RemoveDirectoryFromCache(InDirectory);
- throw;
- }
-
- return fileExisted;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t)
-// Purpose: Undeletes a file, if it exists, returning true if
-// the file existed.
-// Created: 2003/10/21
-//
-// --------------------------------------------------------------------------
-bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory)
-{
- // Essential checks!
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // Find the directory the file is in (will exception if it fails)
- BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
-
- // Setup flags
- bool fileExisted = false;
- bool madeChanges = false;
-
- // Count of deleted blocks
- int64_t blocksDel = 0;
-
- try
- {
- // Iterate through directory, only looking at files which have been deleted
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *e = 0;
- while((e = i.Next(BackupStoreDirectory::Entry::Flags_File |
- BackupStoreDirectory::Entry::Flags_Deleted, 0)) != 0)
- {
- // Compare name
- if(e->GetObjectID() == ObjectID)
- {
- // Check that it's definitely already deleted
- ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
- // Clear deleted flag
- e->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
- // Mark as made a change
- madeChanges = true;
- blocksDel -= e->GetSizeInBlocks();
-
- // Is this the last version?
- if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
- {
- // Yes. It's been found.
- fileExisted = true;
- }
- }
- }
-
- // Save changes?
- if(madeChanges)
- {
- // Save the directory back
- SaveDirectory(dir, InDirectory);
-
- // Modify the store info, and write
- mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel);
-
- // Maybe postponed save of store info
- SaveStoreInfo();
- }
- }
- catch(...)
- {
- RemoveDirectoryFromCache(InDirectory);
- throw;
- }
-
- return fileExisted;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::RemoveDirectoryFromCache(int64_t)
-// Purpose: Remove directory from cache
-// Created: 2003/09/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID)
-{
- std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
- if(item != mDirectoryCache.end())
- {
- // Delete this cached object
- delete item->second;
- // Erase the entry form the map
- mDirectoryCache.erase(item);
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t)
-// Purpose: Save directory back to disc, update time in cache
-// Created: 2003/09/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(rDir.GetObjectID() != ObjectID)
- {
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-
- try
- {
- // Write to disc, adjust size in store info
- std::string dirfn;
- MakeObjectFilename(ObjectID, dirfn);
- {
- RaidFileWrite writeDir(mStoreDiscSet, dirfn);
- writeDir.Open(true /* allow overwriting */);
-
- BufferedWriteStream buffer(writeDir);
- rDir.WriteToStream(buffer);
- buffer.Flush();
-
- // get the disc usage (must do this before commiting it)
- int64_t dirSize = writeDir.GetDiscUsageInBlocks();
-
- // Commit directory
- writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
-
- // Make sure the size of the directory is available for writing the dir back
- ASSERT(dirSize > 0);
- int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks();
- mapStoreInfo->ChangeBlocksUsed(sizeAdjustment);
- mapStoreInfo->ChangeBlocksInDirectories(sizeAdjustment);
- // Update size stored in directory
- rDir.SetUserInfo1_SizeInBlocks(dirSize);
- }
- // Refresh revision ID in cache
- {
- int64_t revid = 0;
- if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid))
- {
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
- rDir.SetRevisionID(revid);
- }
- }
- catch(...)
- {
- // Remove it from the cache if anything went wrong
- RemoveDirectoryFromCache(ObjectID);
- throw;
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::AddDirectory(int64_t,
-// const BackupStoreFilename &, bool &)
-// Purpose: Creates a directory (or just returns the ID of an
-// existing one). rAlreadyExists set appropraitely.
-// Created: 2003/09/04
-//
-// --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // Flags as not already existing
- rAlreadyExists = false;
-
- // Get the directory we want to modify
- BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
-
- // Scan the directory for the name (only looking for directories which already exist)
- {
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *en = 0;
- while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING,
- BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories
- {
- if(en->GetName() == rFilename)
- {
- // Already exists
- rAlreadyExists = true;
- return en->GetObjectID();
- }
- }
- }
-
- // Allocate the next ID
- int64_t id = AllocateObjectID();
-
- // Create an empty directory with the given attributes on disc
- std::string fn;
- MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
- {
- BackupStoreDirectory emptyDir(id, InDirectory);
- // add the atttribues
- emptyDir.SetAttributes(Attributes, AttributesModTime);
-
- // Write...
- RaidFileWrite dirFile(mStoreDiscSet, fn);
- dirFile.Open(false /* no overwriting */);
- emptyDir.WriteToStream(dirFile);
- // Get disc usage, before it's commited
- int64_t dirSize = dirFile.GetDiscUsageInBlocks();
- // Commit the file
- dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
-
- // Make sure the size of the directory is added to the usage counts in the info
- ASSERT(dirSize > 0);
- mapStoreInfo->ChangeBlocksUsed(dirSize);
- mapStoreInfo->ChangeBlocksInDirectories(dirSize);
- // Not added to cache, so don't set the size in the directory
- }
-
- // Then add it into the parent directory
- try
- {
- dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
- SaveDirectory(dir, InDirectory);
-
- // Increment reference count on the new directory to one
- mapRefCount->AddReference(id);
- }
- catch(...)
- {
- // Back out on adding that directory
- RaidFileWrite del(mStoreDiscSet, fn);
- del.Delete();
-
- // Remove this entry from the cache
- RemoveDirectoryFromCache(InDirectory);
-
- // Don't worry about the incremented number in the store info
- throw;
- }
-
- // Save the store info (may not be postponed)
- mapStoreInfo->AdjustNumDirectories(1);
- SaveStoreInfo(false);
-
- // tell caller what the ID was
- return id;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool)
-// Purpose: Recusively deletes a directory (or undeletes if Undelete = true)
-// Created: 2003/10/21
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete)
-{
- // Essential checks!
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // Containing directory
- int64_t InDirectory = 0;
-
- // Count of blocks deleted
- int64_t blocksDeleted = 0;
-
- try
- {
- // Get the directory that's to be deleted
- {
- // In block, because dir may not be valid after the delete directory call
- BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
-
- // Store the directory it's in for later
- InDirectory = dir.GetContainerID();
-
- // Depth first delete of contents
- DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete);
- }
-
- // Remove the entry from the directory it's in
- ASSERT(InDirectory != 0);
- BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory));
-
- BackupStoreDirectory::Iterator i(parentDir);
- BackupStoreDirectory::Entry *en = 0;
- while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
- Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete)
- {
- if(en->GetObjectID() == ObjectID)
- {
- // This is the one to delete
- if(Undelete)
- {
- en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
- }
- else
- {
- en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
- }
-
- // Save it
- SaveDirectory(parentDir, InDirectory);
-
- // Done
- break;
- }
- }
-
- // Update blocks deleted count
- mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted));
- mapStoreInfo->AdjustNumDirectories(-1);
- SaveStoreInfo(false);
- }
- catch(...)
- {
- RemoveDirectoryFromCache(InDirectory);
- throw;
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t)
-// Purpose: Private. Deletes a directory depth-first recusively.
-// Created: 2003/10/21
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete)
-{
- try
- {
- // Does things carefully to avoid using a directory in the cache after recursive call
- // because it may have been deleted.
-
- // Do sub directories
- {
- // Get the directory...
- BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
-
- // Then scan it for directories
- std::vector<int64_t> subDirs;
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *en = 0;
- if(Undelete)
- {
- while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs
- BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0)
- {
- // Store the directory ID.
- subDirs.push_back(en->GetObjectID());
- }
- }
- else
- {
- while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only
- BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones
- {
- // Store the directory ID.
- subDirs.push_back(en->GetObjectID());
- }
- }
-
- // Done with the directory for now. Recurse to sub directories
- for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i)
- {
- DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete);
- }
- }
-
- // Then, delete the files. Will need to load the directory again because it might have
- // been removed from the cache.
- {
- // Get the directory...
- BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
-
- // Changes made?
- bool changesMade = false;
-
- // Run through files
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *en = 0;
-
- while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
- Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete)
- {
- // Add/remove the deleted flags
- if(Undelete)
- {
- en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
- }
- else
- {
- en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
- }
-
- // Keep count of the deleted blocks
- if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
- {
- rBlocksDeletedOut += en->GetSizeInBlocks();
- }
-
- // Did something
- changesMade = true;
- }
-
- // Save the directory
- if(changesMade)
- {
- SaveDirectory(dir, ObjectID);
- }
- }
- }
- catch(...)
- {
- RemoveDirectoryFromCache(ObjectID);
- throw;
- }
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t)
-// Purpose: Change the attributes of a directory
-// Created: 2003/09/06
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- try
- {
- // Get the directory we want to modify
- BackupStoreDirectory &dir(GetDirectoryInternal(Directory));
-
- // Set attributes
- dir.SetAttributes(Attributes, AttributesModTime);
-
- // Save back
- SaveDirectory(dir, Directory);
- }
- catch(...)
- {
- RemoveDirectoryFromCache(Directory);
- throw;
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t)
-// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't.
-// Created: 2003/09/06
-//
-// --------------------------------------------------------------------------
-bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- try
- {
- // Get the directory we want to modify
- BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
-
- // Find the file entry
- BackupStoreDirectory::Entry *en = 0;
- // Iterate through current versions of files, only
- BackupStoreDirectory::Iterator i(dir);
- while((en = i.Next(
- BackupStoreDirectory::Entry::Flags_File,
- BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)
- ) != 0)
- {
- if(en->GetName() == rFilename)
- {
- // Set attributes
- en->SetAttributes(Attributes, AttributesHash);
-
- // Tell caller the object ID
- rObjectIDOut = en->GetObjectID();
-
- // Done
- break;
- }
- }
- if(en == 0)
- {
- // Didn't find it
- return false;
- }
-
- // Save back
- SaveDirectory(dir, InDirectory);
- }
- catch(...)
- {
- RemoveDirectoryFromCache(InDirectory);
- throw;
- }
-
- // Changed, everything OK
- return true;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::ObjectExists(int64_t)
-// Purpose: Test to see if an object of this ID exists in the store
-// Created: 2003/09/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- // Note that we need to allow object IDs a little bit greater than the last one in the store info,
- // because the store info may not have got saved in an error condition. Max greater ID is
- // STORE_INFO_SAVE_DELAY in this case, *2 to be safe.
- if(ObjectID <= 0 || ObjectID > (mapStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2)))
- {
- // Obviously bad object ID
- return false;
- }
-
- // Test to see if it exists on the disc
- std::string filename;
- MakeObjectFilename(ObjectID, filename);
- if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
- {
- // RaidFile reports no file there
- return false;
- }
-
- // Do we need to be more specific?
- if(MustBe != ObjectExists_Anything)
- {
- // Open the file
- std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename));
-
- // Read the first integer
- u_int32_t magic;
- if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */))
- {
- // Failed to get any bytes, must have failed
- return false;
- }
-
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0)
- {
- // Old version detected
- return true;
- }
-#endif
-
- // Right one?
- u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE;
-
- // Check
- if(ntohl(magic) != requiredMagic)
- {
- return false;
- }
-
- // File is implicitly closed
- }
-
- return true;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::OpenObject(int64_t)
-// Purpose: Opens an object
-// Created: 2003/09/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<IOStream> BackupStoreContext::OpenObject(int64_t ObjectID)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- // Attempt to open the file
- std::string fn;
- MakeObjectFilename(ObjectID, fn);
- return std::auto_ptr<IOStream>(RaidFileRead::Open(mStoreDiscSet, fn).release());
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::GetClientStoreMarker()
-// Purpose: Retrieve the client store marker
-// Created: 2003/10/29
-//
-// --------------------------------------------------------------------------
-int64_t BackupStoreContext::GetClientStoreMarker()
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- return mapStoreInfo->GetClientStoreMarker();
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &)
-// Purpose: Get disc usage info from store info
-// Created: 1/1/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- rBlocksUsed = mapStoreInfo->GetBlocksUsed();
- rBlocksSoftLimit = mapStoreInfo->GetBlocksSoftLimit();
- rBlocksHardLimit = mapStoreInfo->GetBlocksHardLimit();
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::HardLimitExceeded()
-// Purpose: Returns true if the hard limit has been exceeded
-// Created: 1/1/04
-//
-// --------------------------------------------------------------------------
-bool BackupStoreContext::HardLimitExceeded()
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- return mapStoreInfo->GetBlocksUsed() > mapStoreInfo->GetBlocksHardLimit();
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::SetClientStoreMarker(int64_t)
-// Purpose: Sets the client store marker, and commits it to disc
-// Created: 2003/10/29
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker)
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- mapStoreInfo->SetClientStoreMarker(ClientStoreMarker);
- SaveStoreInfo(false /* don't delay saving this */);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-// Purpose: Move an object (and all objects with the same name) from one directory to another
-// Created: 12/11/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject)
-{
- if(mReadOnly)
- {
- THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
- }
-
- // Should deleted files be excluded when checking for the existance of objects with the target name?
- int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject)
- ?(BackupStoreDirectory::Entry::Flags_Deleted)
- :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
-
- // Special case if the directories are the same...
- if(MoveFromDirectory == MoveToDirectory)
- {
- try
- {
- // Get the first directory
- BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory));
-
- // Find the file entry
- BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID);
-
- // Error if not found
- if(en == 0)
- {
- THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
- }
-
- // Check the new name doens't already exist (optionally ignoring deleted files)
- {
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *c = 0;
- while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
- {
- if(c->GetName() == rNewFilename)
- {
- THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
- }
- }
- }
-
- // Need to get all the entries with the same name?
- if(MoveAllWithSameName)
- {
- // Iterate through the directory, copying all with matching names
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *c = 0;
- while((c = i.Next()) != 0)
- {
- if(c->GetName() == en->GetName())
- {
- // Rename this one
- c->SetName(rNewFilename);
- }
- }
- }
- else
- {
- // Just copy this one
- en->SetName(rNewFilename);
- }
-
- // Save the directory back
- SaveDirectory(dir, MoveFromDirectory);
- }
- catch(...)
- {
- RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same
- throw;
- }
-
- return;
- }
-
- // Got to be careful how this is written, as we can't guarentte that if we have two
- // directories open, the first won't be deleted as the second is opened. (cache)
-
- // List of entries to move
- std::vector<BackupStoreDirectory::Entry *> moving;
-
- // list of directory IDs which need to have containing dir id changed
- std::vector<int64_t> dirsToChangeContainingID;
-
- try
- {
- // First of all, get copies of the entries to move to the to directory.
-
- {
- // Get the first directory
- BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
-
- // Find the file entry
- BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID);
-
- // Error if not found
- if(en == 0)
- {
- THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
- }
-
- // Need to get all the entries with the same name?
- if(MoveAllWithSameName)
- {
- // Iterate through the directory, copying all with matching names
- BackupStoreDirectory::Iterator i(from);
- BackupStoreDirectory::Entry *c = 0;
- while((c = i.Next()) != 0)
- {
- if(c->GetName() == en->GetName())
- {
- // Copy
- moving.push_back(new BackupStoreDirectory::Entry(*c));
-
- // Check for containing directory correction
- if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID());
- }
- }
- ASSERT(!moving.empty());
- }
- else
- {
- // Just copy this one
- moving.push_back(new BackupStoreDirectory::Entry(*en));
-
- // Check for containing directory correction
- if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID());
- }
- }
-
- // Secondly, insert them into the to directory, and save it
-
- {
- // To directory
- BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
-
- // Check the new name doens't already exist
- {
- BackupStoreDirectory::Iterator i(to);
- BackupStoreDirectory::Entry *c = 0;
- while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
- {
- if(c->GetName() == rNewFilename)
- {
- THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
- }
- }
- }
-
- // Copy the entries into it, changing the name as we go
- for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
- {
- BackupStoreDirectory::Entry *en = (*i);
- en->SetName(rNewFilename);
- to.AddEntry(*en); // adds copy
- }
-
- // Save back
- SaveDirectory(to, MoveToDirectory);
- }
-
- // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory
- try
- {
- // Get directory
- BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
-
- // Delete each one
- for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
- {
- from.DeleteEntry((*i)->GetObjectID());
- }
-
- // Save back
- SaveDirectory(from, MoveFromDirectory);
- }
- catch(...)
- {
- // UNDO modification to To directory
-
- // Get directory
- BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
-
- // Delete each one
- for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
- {
- to.DeleteEntry((*i)->GetObjectID());
- }
-
- // Save back
- SaveDirectory(to, MoveToDirectory);
-
- // Throw the error
- throw;
- }
-
- // Finally... for all the directories we moved, modify their containing directory ID
- for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
- {
- // Load the directory
- BackupStoreDirectory &change(GetDirectoryInternal(*i));
-
- // Modify containing dir ID
- change.SetContainerID(MoveToDirectory);
-
- // Save it back
- SaveDirectory(change, *i);
- }
- }
- catch(...)
- {
- // Make sure directories aren't in the cache, as they may have been modified
- RemoveDirectoryFromCache(MoveToDirectory);
- RemoveDirectoryFromCache(MoveFromDirectory);
- for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
- {
- RemoveDirectoryFromCache(*i);
- }
-
- while(!moving.empty())
- {
- delete moving.back();
- moving.pop_back();
- }
- throw;
- }
-
- // Clean up
- while(!moving.empty())
- {
- delete moving.back();
- moving.pop_back();
- }
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreContext::GetBackupStoreInfo()
-// Purpose: Return the backup store info object, exception if it isn't loaded
-// Created: 19/4/04
-//
-// --------------------------------------------------------------------------
-const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const
-{
- if(mapStoreInfo.get() == 0)
- {
- THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
- }
-
- return *(mapStoreInfo.get());
-}
-
-
Deleted: box/trunk/bin/bbstored/BackupStoreContext.h
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/bin/bbstored/BackupStoreContext.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,186 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreContext.h
-// Purpose: Context for backup store server
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPCONTEXT__H
-#define BACKUPCONTEXT__H
-
-#include <string>
-#include <map>
-#include <memory>
-
-#include "BackupStoreRefCountDatabase.h"
-#include "NamedLock.h"
-#include "ProtocolObject.h"
-#include "Utils.h"
-
-class BackupStoreDirectory;
-class BackupStoreFilename;
-class BackupStoreDaemon;
-class BackupStoreInfo;
-class IOStream;
-class BackupProtocolObject;
-class StreamableMemBlock;
-
-class HousekeepingInterface
-{
- public:
- virtual ~HousekeepingInterface() { }
- virtual void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) = 0;
-};
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupStoreContext
-// Purpose: Context for backup store server
-// Created: 2003/08/20
-//
-// --------------------------------------------------------------------------
-class BackupStoreContext
-{
-public:
- BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon);
- ~BackupStoreContext();
-private:
- BackupStoreContext(const BackupStoreContext &rToCopy);
-public:
-
- void ReceivedFinishCommand();
- void CleanUp();
-
- int32_t GetClientID() {return mClientID;}
-
- enum
- {
- Phase_START = 0,
- Phase_Version = 0,
- Phase_Login = 1,
- Phase_Commands = 2
- };
-
- int GetPhase() const {return mProtocolPhase;}
- void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;}
-
- // Read only locking
- bool SessionIsReadOnly() {return mReadOnly;}
- bool AttemptToGetWriteLock();
-
- void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;}
- bool GetClientHasAccount() const {return mClientHasAccount;}
- const std::string &GetStoreRoot() const {return mStoreRoot;}
- int GetStoreDiscSet() const {return mStoreDiscSet;}
-
- // Store info
- void LoadStoreInfo();
- void SaveStoreInfo(bool AllowDelay = true);
- const BackupStoreInfo &GetBackupStoreInfo() const;
-
- // Client marker
- int64_t GetClientStoreMarker();
- void SetClientStoreMarker(int64_t ClientStoreMarker);
-
- // Usage information
- void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit);
- bool HardLimitExceeded();
-
- // Reading directories
- // --------------------------------------------------------------------------
- //
- // Function
- // Name: BackupStoreContext::GetDirectory(int64_t)
- // Purpose: Return a reference to a directory. Valid only until the
- // next time a function which affects directories is called.
- // Mainly this funciton, and creation of files.
- // Created: 2003/09/02
- //
- // --------------------------------------------------------------------------
- const BackupStoreDirectory &GetDirectory(int64_t ObjectID)
- {
- // External callers aren't allowed to change it -- this function
- // merely turns the the returned directory const.
- return GetDirectoryInternal(ObjectID);
- }
-
- // Manipulating files/directories
- int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions);
- int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists);
- void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime);
- bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut);
- bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut);
- bool UndeleteFile(int64_t ObjectID, int64_t InDirectory);
- void DeleteDirectory(int64_t ObjectID, bool Undelete = false);
- void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject);
-
- // Manipulating objects
- enum
- {
- ObjectExists_Anything = 0,
- ObjectExists_File = 1,
- ObjectExists_Directory = 2
- };
- bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything);
- std::auto_ptr<IOStream> OpenObject(int64_t ObjectID);
-
- // Info
- int32_t GetClientID() const {return mClientID;}
-
-private:
- void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false);
- BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID);
- void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID);
- void RemoveDirectoryFromCache(int64_t ObjectID);
- void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
- int64_t AllocateObjectID();
-
- int32_t mClientID;
- HousekeepingInterface &mrDaemon;
- int mProtocolPhase;
- bool mClientHasAccount;
- std::string mStoreRoot; // has final directory separator
- int mStoreDiscSet;
- bool mReadOnly;
- NamedLock mWriteLock;
- int mSaveStoreInfoDelay; // how many times to delay saving the store info
-
- // Store info
- std::auto_ptr<BackupStoreInfo> mapStoreInfo;
-
- // Refcount database
- std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
-
- // Directory cache
- std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
-
-public:
- class TestHook
- {
- public:
- virtual std::auto_ptr<ProtocolObject> StartCommand(BackupProtocolObject&
- rCommand) = 0;
- virtual ~TestHook() { }
- };
- void SetTestHook(TestHook& rTestHook)
- {
- mpTestHook = &rTestHook;
- }
- std::auto_ptr<ProtocolObject> StartCommandHook(BackupProtocolObject& rCommand)
- {
- if(mpTestHook)
- {
- return mpTestHook->StartCommand(rCommand);
- }
- return std::auto_ptr<ProtocolObject>();
- }
-
-private:
- TestHook* mpTestHook;
-};
-
-#endif // BACKUPCONTEXT__H
-
Deleted: box/trunk/bin/bbstored/Makefile.extra
===================================================================
--- box/trunk/bin/bbstored/Makefile.extra 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/bin/bbstored/Makefile.extra 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,9 +0,0 @@
-
-MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
-
-GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt
-
-# AUTOGEN SEEDING
-autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h: $(MAKEPROTOCOL) backupprotocol.txt
- $(_PERL) $(GEN_CMD_SRV)
-
Deleted: box/trunk/bin/bbstored/backupprotocol.txt
===================================================================
--- box/trunk/bin/bbstored/backupprotocol.txt 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/bin/bbstored/backupprotocol.txt 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,235 +0,0 @@
-#
-# backup protocol definition
-#
-
-Name Backup
-IdentString Box-Backup:v=C
-ServerContextClass BackupStoreContext BackupStoreContext.h
-
-ClientType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h
-ServerType Filename BackupStoreFilename BackupStoreFilename.h
-
-ImplementLog Server syslog
-ImplementLog Client syslog
-ImplementLog Client file
-
-LogTypeToText Client Filename \"%s\" VAR.GetClearFilename().c_str()
-
-BEGIN_OBJECTS
-
-# -------------------------------------------------------------------------------------
-# Session commands
-# -------------------------------------------------------------------------------------
-
-Error 0 IsError(Type,SubType) Reply
- int32 Type
- int32 SubType
- CONSTANT ErrorType 1000
- CONSTANT Err_WrongVersion 1
- CONSTANT Err_NotInRightProtocolPhase 2
- CONSTANT Err_BadLogin 3
- CONSTANT Err_CannotLockStoreForWriting 4
- CONSTANT Err_SessionReadOnly 5
- CONSTANT Err_FileDoesNotVerify 6
- CONSTANT Err_DoesNotExist 7
- CONSTANT Err_DirectoryAlreadyExists 8
- CONSTANT Err_CannotDeleteRoot 9
- CONSTANT Err_TargetNameExists 10
- CONSTANT Err_StorageLimitExceeded 11
- CONSTANT Err_DiffFromFileDoesNotExist 12
- CONSTANT Err_DoesNotExistInDirectory 13
- CONSTANT Err_PatchConsistencyError 14
- CONSTANT Err_MultiplyReferencedObject 15
-
-Version 1 Command(Version) Reply
- int32 Version
-
-
-Login 2 Command(LoginConfirmed)
- int32 ClientID
- int32 Flags
- CONSTANT Flags_ReadOnly 1
-
-
-LoginConfirmed 3 Reply
- int64 ClientStoreMarker
- int64 BlocksUsed
- int64 BlocksSoftLimit
- int64 BlocksHardLimit
-
-
-Finished 4 Command(Finished) Reply EndsConversation
-
-
-# generic success object
-Success 5 Reply
- int64 ObjectID
-
-
-SetClientStoreMarker 6 Command(Success)
- int64 ClientStoreMarker
-
-
-# -------------------------------------------------------------------------------------
-# Generic object commands
-# -------------------------------------------------------------------------------------
-
-GetObject 10 Command(Success)
- int64 ObjectID
- CONSTANT NoObject 0
- # reply has stream following, if ObjectID != NoObject
-
-
-MoveObject 11 Command(Success)
- int64 ObjectID
- int64 MoveFromDirectory
- int64 MoveToDirectory
- int32 Flags
- Filename NewFilename
-
- CONSTANT Flags_MoveAllWithSameName 1
- CONSTANT Flags_AllowMoveOverDeletedObject 2
-
-# consider this an object command as, although it deals with directory entries,
-# it's not specific to either a file or a directory
-
-
-GetObjectName 12 Command(ObjectName)
- int64 ObjectID
- int64 ContainingDirectoryID
- CONSTANT ObjectID_DirectoryOnly 0
-
- # set ObjectID to ObjectID_DirectoryOnly to only get info on the directory
-
-
-ObjectName 13 Reply
- int32 NumNameElements
- int64 ModificationTime
- int64 AttributesHash
- int16 Flags
- # NumNameElements is zero if the object doesn't exist
- CONSTANT NumNameElements_ObjectDoesntExist 0
- # a stream of Filename objects follows, if and only if NumNameElements > 0
-
-
-# -------------------------------------------------------------------------------------
-# Directory commands
-# -------------------------------------------------------------------------------------
-
-CreateDirectory 20 Command(Success) StreamWithCommand
- int64 ContainingDirectoryID
- int64 AttributesModTime
- Filename DirectoryName
- # stream following containing attributes
-
-
-ListDirectory 21 Command(Success)
- int64 ObjectID
- int16 FlagsMustBeSet
- int16 FlagsNotToBeSet
- bool SendAttributes
- # make sure these flags are synced with those in BackupStoreDirectory
- CONSTANT Flags_INCLUDE_EVERYTHING -1
- CONSTANT Flags_EXCLUDE_NOTHING 0
- CONSTANT Flags_EXCLUDE_EVERYTHING 15
- CONSTANT Flags_File 1
- CONSTANT Flags_Dir 2
- CONSTANT Flags_Deleted 4
- CONSTANT Flags_OldVersion 8
- # make sure this is the same as in BackupStoreConstants.h
- CONSTANT RootDirectory 1
-
- # reply has stream following Success object, containing a stored BackupStoreDirectory
-
-
-ChangeDirAttributes 22 Command(Success) StreamWithCommand
- int64 ObjectID
- int64 AttributesModTime
- # stream following containing attributes
-
-
-DeleteDirectory 23 Command(Success)
- int64 ObjectID
-
-UndeleteDirectory 24 Command(Success)
- int64 ObjectID
- # may not have exactly the desired effect if files within in have been deleted before the directory was deleted.
-
-
-# -------------------------------------------------------------------------------------
-# File commands
-# -------------------------------------------------------------------------------------
-
-StoreFile 30 Command(Success) StreamWithCommand
- int64 DirectoryObjectID
- int64 ModificationTime
- int64 AttributesHash
- int64 DiffFromFileID # 0 if the file is not a diff
- Filename Filename
- # then send a stream containing the encoded file
-
-
-GetFile 31 Command(Success)
- int64 InDirectory
- int64 ObjectID
- # error returned if not a file, or does not exist
- # reply has stream following, containing an encoded file IN STREAM ORDER
- # (use GetObject to get it in file order)
-
-
-SetReplacementFileAttributes 32 Command(Success) StreamWithCommand
- int64 InDirectory
- int64 AttributesHash
- Filename Filename
- # stream follows containing attributes
-
-
-DeleteFile 33 Command(Success)
- int64 InDirectory
- Filename Filename
- # will return 0 if the object couldn't be found in the specified directory
-
-
-GetBlockIndexByID 34 Command(Success)
- int64 ObjectID
-
- # stream of the block index follows the reply
- # returns an error if the object didn't exist
-
-
-GetBlockIndexByName 35 Command(Success)
- int64 InDirectory
- Filename Filename
-
- # Success object contains the found ID -- or 0 if the entry wasn't found in the directory
- # stream of the block index follows the reply if found ID != 0
-
-
-UndeleteFile 36 Command(Success)
- int64 InDirectory
- int64 ObjectID
- # will return 0 if the object couldn't be found in the specified directory
-
-
-# -------------------------------------------------------------------------------------
-# Information commands
-# -------------------------------------------------------------------------------------
-
-GetAccountUsage 40 Command(AccountUsage)
- # no data members
-
-AccountUsage 41 Reply
- int64 BlocksUsed
- int64 BlocksInOldFiles
- int64 BlocksInDeletedFiles
- int64 BlocksInDirectories
- int64 BlocksSoftLimit
- int64 BlocksHardLimit
- int32 BlockSize
-
-GetIsAlive 42 Command(IsAlive)
- # no data members
-
-IsAlive 43 Reply
- # no data members
-
Deleted: box/trunk/lib/backupclient/BackupClientFileAttributes.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupClientFileAttributes.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupClientFileAttributes.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,1188 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupClientFileAttributes.cpp
-// Purpose: Storage of file attributes
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#ifdef HAVE_UNISTD_H
- #include <unistd.h>
-#endif
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <limits.h>
-
-#include <algorithm>
-#include <cstring>
-#include <new>
-#include <vector>
-
-#ifdef HAVE_SYS_XATTR_H
-#include <cerrno>
-#include <sys/xattr.h>
-#endif
-
-#include <cstring>
-
-#include "BackupClientFileAttributes.h"
-#include "CommonException.h"
-#include "FileModificationTime.h"
-#include "BoxTimeToUnix.h"
-#include "BackupStoreException.h"
-#include "CipherContext.h"
-#include "CipherBlowfish.h"
-#include "MD5Digest.h"
-
-#include "MemLeakFindOn.h"
-
-// set packing to one byte
-#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
-#include "BeginStructPackForWire.h"
-#else
-BEGIN_STRUCTURE_PACKING_FOR_WIRE
-#endif
-
-#define ATTRIBUTETYPE_GENERIC_UNIX 1
-
-#define ATTRIBUTE_ENCODING_BLOWFISH 2
-
-typedef struct
-{
- int32_t AttributeType;
- u_int32_t UID;
- u_int32_t GID;
- u_int64_t ModificationTime;
- u_int64_t AttrModificationTime;
- u_int32_t UserDefinedFlags;
- u_int32_t FileGenerationNumber;
- u_int16_t Mode;
- // Symbolic link filename may follow
- // Extended attribute (xattr) information may follow, format is:
- // u_int32_t Size of extended attribute block (excluding this word)
- // For each of NumberOfAttributes (sorted by AttributeName):
- // u_int16_t AttributeNameLength
- // char AttributeName[AttributeNameLength]
- // u_int32_t AttributeValueLength
- // unsigned char AttributeValue[AttributeValueLength]
- // AttributeName is 0 terminated, AttributeValue is not (and may be binary data)
-} attr_StreamFormat;
-
-// This has wire packing so it's compatible across platforms
-// Use wider than necessary sizes, just to be careful.
-typedef struct
-{
- int32_t uid, gid, mode;
- #ifdef WIN32
- int64_t fileCreationTime;
- #endif
-} attributeHashData;
-
-// Use default packing
-#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
-#include "EndStructPackForWire.h"
-#else
-END_STRUCTURE_PACKING_FOR_WIRE
-#endif
-
-
-#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256
-
-// Hide private static variables from the rest of the world
-// -- don't put them as static class variables to avoid openssl/evp.h being
-// included all over the project.
-namespace
-{
- CipherContext sBlowfishEncrypt;
- CipherContext sBlowfishDecrypt;
- uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH];
- int sAttributeHashSecretLength = 0;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::BackupClientFileAttributes()
-// Purpose: Default constructor
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-BackupClientFileAttributes::BackupClientFileAttributes()
- : mpClearAttributes(0)
-{
- ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &)
-// Purpose: Copy constructor
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy)
- : StreamableMemBlock(rToCopy), // base class does the hard work
- mpClearAttributes(0)
-{
-}
-BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy)
- : StreamableMemBlock(rToCopy), // base class does the hard work
- mpClearAttributes(0)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::~BackupClientFileAttributes()
-// Purpose: Destructor
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-BackupClientFileAttributes::~BackupClientFileAttributes()
-{
- if(mpClearAttributes)
- {
- delete mpClearAttributes;
- mpClearAttributes = 0;
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &)
-// Purpose: Assignment operator
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr)
-{
- StreamableMemBlock::Set(rAttr);
- RemoveClear(); // make sure no decrypted version held
- return *this;
-}
-// Assume users play nice
-BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr)
-{
- StreamableMemBlock::Set(rAttr);
- RemoveClear(); // make sure no decrypted version held
- return *this;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &)
-// Purpose: Comparison operator
-// Created: 2003/10/09
-//
-// --------------------------------------------------------------------------
-bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const
-{
- EnsureClearAvailable();
- rAttr.EnsureClearAvailable();
-
- return mpClearAttributes->operator==(*rAttr.mpClearAttributes);
-}
-// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data.
-/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const
-{
- StreamableMemBlock *pDecoded = 0;
-
- try
- {
- EnsureClearAvailable();
- StreamableMemBlock *pDecoded = MakeClear(rAttr);
-
- // Compare using clear version
- bool compared = mpClearAttributes->operator==(rAttr);
-
- // Delete temporary
- delete pDecoded;
-
- return compared;
- }
- catch(...)
- {
- delete pDecoded;
- throw;
- }
-}*/
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
-// Purpose: Compare, optionally ignoring the attribute
-// modification time and/or modification time, and some
-// data which is irrelevant in practise (eg file
-// generation number)
-// Created: 10/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr,
- bool IgnoreAttrModTime, bool IgnoreModTime) const
-{
- EnsureClearAvailable();
- rAttr.EnsureClearAvailable();
-
- // Check sizes are the same, as a first check
- if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
- {
- BOX_TRACE("Attribute Compare: Attributes objects are "
- "different sizes, cannot compare them: local " <<
- mpClearAttributes->GetSize() << " bytes, remote " <<
- rAttr.mpClearAttributes->GetSize() << " bytes");
- return false;
- }
-
- // Then check the elements of the two things
- // Bytes are checked in network order, but this doesn't matter as we're only checking for equality.
- attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
- attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
-
- #define COMPARE(attribute, message) \
- if (a1->attribute != a2->attribute) \
- { \
- BOX_TRACE("Attribute Compare: " << message << " differ: " \
- "local " << ntoh(a1->attribute) << ", " \
- "remote " << ntoh(a2->attribute)); \
- return false; \
- }
- COMPARE(AttributeType, "Attribute types");
- COMPARE(UID, "UIDs");
- COMPARE(GID, "GIDs");
- COMPARE(UserDefinedFlags, "User-defined flags");
- COMPARE(Mode, "Modes");
-
- if(!IgnoreModTime)
- {
- uint64_t t1 = box_ntoh64(a1->ModificationTime);
- uint64_t t2 = box_ntoh64(a2->ModificationTime);
- time_t s1 = BoxTimeToSeconds(t1);
- time_t s2 = BoxTimeToSeconds(t2);
- if(s1 != s2)
- {
- BOX_TRACE("Attribute Compare: File modification "
- "times differ: local " <<
- FormatTime(t1, true) << " (" << s1 << "), "
- "remote " <<
- FormatTime(t2, true) << " (" << s2 << ")");
- return false;
- }
- }
-
- if(!IgnoreAttrModTime)
- {
- uint64_t t1 = box_ntoh64(a1->AttrModificationTime);
- uint64_t t2 = box_ntoh64(a2->AttrModificationTime);
- time_t s1 = BoxTimeToSeconds(t1);
- time_t s2 = BoxTimeToSeconds(t2);
- if(s1 != s2)
- {
- BOX_TRACE("Attribute Compare: Attribute modification "
- "times differ: local " <<
- FormatTime(t1, true) << " (" << s1 << "), "
- "remote " <<
- FormatTime(t2, true) << " (" << s2 << ")");
- return false;
- }
- }
-
- // Check symlink string?
- unsigned int size = mpClearAttributes->GetSize();
- if(size > sizeof(attr_StreamFormat))
- {
- // Symlink strings don't match. This also compares xattrs
- int datalen = size - sizeof(attr_StreamFormat);
-
- if(::memcmp(a1 + 1, a2 + 1, datalen) != 0)
- {
- std::string s1((char *)(a1 + 1), datalen);
- std::string s2((char *)(a2 + 1), datalen);
- BOX_TRACE("Attribute Compare: Symbolic link target "
- "or extended attributes differ: "
- "local " << PrintEscapedBinaryData(s1) << ", "
- "remote " << PrintEscapedBinaryData(s2));
- return false;
- }
- }
-
- // Passes all test, must be OK
- return true;
-}
-
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::ReadAttributes(
-// const char *Filename, bool ZeroModificationTimes,
-// box_time_t *pModTime, box_time_t *pAttrModTime,
-// int64_t *pFileSize, InodeRefType *pInodeNumber,
-// bool *pHasMultipleLinks)
-// Purpose: Read the attributes of the file, and store them
-// ready for streaming. Optionally retrieve the
-// modification time and attribute modification time.
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::ReadAttributes(const char *Filename,
- bool ZeroModificationTimes, box_time_t *pModTime,
- box_time_t *pAttrModTime, int64_t *pFileSize,
- InodeRefType *pInodeNumber, bool *pHasMultipleLinks)
-{
- StreamableMemBlock *pnewAttr = 0;
- try
- {
- EMU_STRUCT_STAT st;
- if(EMU_LSTAT(Filename, &st) != 0)
- {
- BOX_LOG_SYS_ERROR("Failed to stat file: '" <<
- Filename << "'");
- THROW_EXCEPTION(CommonException, OSFileError)
- }
-
- // Modification times etc
- if(pModTime) {*pModTime = FileModificationTime(st);}
- if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);}
- if(pFileSize) {*pFileSize = st.st_size;}
- if(pInodeNumber) {*pInodeNumber = st.st_ino;}
- if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);}
-
- pnewAttr = new StreamableMemBlock;
-
- FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes);
-
-#ifndef WIN32
- // Is it a link?
- if((st.st_mode & S_IFMT) == S_IFLNK)
- {
- FillAttributesLink(*pnewAttr, Filename, st);
- }
-#endif
-
- FillExtendedAttr(*pnewAttr, Filename);
-
-#ifdef WIN32
- //this is to catch those problems with invalid time stamps stored...
- //need to find out the reason why - but also a catch as well.
-
- attr_StreamFormat *pattr =
- (attr_StreamFormat*)pnewAttr->GetBuffer();
- ASSERT(pattr != 0);
-
- // __time64_t winTime = BoxTimeToSeconds(
- // pnewAttr->ModificationTime);
-
- u_int64_t modTime = box_ntoh64(pattr->ModificationTime);
- box_time_t modSecs = BoxTimeToSeconds(modTime);
- __time64_t winTime = modSecs;
-
- // _MAX__TIME64_T doesn't seem to be defined, but the code below
- // will throw an assertion failure if we exceed it :-)
- // Microsoft says dates up to the year 3000 are valid, which
- // is a bit more than 15 * 2^32. Even that doesn't seem
- // to be true (still aborts), but it can at least hold 2^32.
- if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0)
- {
- BOX_ERROR("Invalid Modification Time caught for "
- "file: '" << Filename << "'");
- pattr->ModificationTime = 0;
- }
-
- modTime = box_ntoh64(pattr->AttrModificationTime);
- modSecs = BoxTimeToSeconds(modTime);
- winTime = modSecs;
-
- if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0)
- {
- BOX_ERROR("Invalid Attribute Modification Time "
- "caught for file: '" << Filename << "'");
- pattr->AttrModificationTime = 0;
- }
-#endif
-
- // Attributes ready. Encrypt into this block
- EncryptAttr(*pnewAttr);
-
- // Store the new attributes
- RemoveClear();
- mpClearAttributes = pnewAttr;
- pnewAttr = 0;
- }
- catch(...)
- {
- // clean up
- delete pnewAttr;
- pnewAttr = 0;
- throw;
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::ReadAttributesLink()
-// Purpose: Private function, handles standard attributes for all objects
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes)
-{
- outputBlock.ResizeBlock(sizeof(attr_StreamFormat));
- attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer();
- ASSERT(pattr != 0);
-
- // Fill in the entries
- pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX);
- pattr->UID = htonl(st.st_uid);
- pattr->GID = htonl(st.st_gid);
- if(ZeroModificationTimes)
- {
- pattr->ModificationTime = 0;
- pattr->AttrModificationTime = 0;
- }
- else
- {
- pattr->ModificationTime = box_hton64(FileModificationTime(st));
- pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st));
- }
- pattr->Mode = htons(st.st_mode);
-
-#ifndef HAVE_STRUCT_STAT_ST_FLAGS
- pattr->UserDefinedFlags = 0;
- pattr->FileGenerationNumber = 0;
-#else
- pattr->UserDefinedFlags = htonl(st.st_flags);
- pattr->FileGenerationNumber = htonl(st.st_gen);
-#endif
-}
-#ifndef WIN32
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::ReadAttributesLink()
-// Purpose: Private function, handles the case where a symbolic link is needed
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st)
-{
- // Make sure we're only called for symbolic links
- ASSERT((st.st_mode & S_IFMT) == S_IFLNK);
-
- // Get the filename the link is linked to
- char linkedTo[PATH_MAX+4];
- int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX);
- if(linkedToSize == -1)
- {
- BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'");
- THROW_EXCEPTION(CommonException, OSFileError);
- }
-
- int oldSize = outputBlock.GetSize();
- outputBlock.ResizeBlock(oldSize+linkedToSize+1);
- char* buffer = static_cast<char*>(outputBlock.GetBuffer());
-
- // Add the path name for the symbolic link, and add 0 termination
- std::memcpy(buffer+oldSize, linkedTo, linkedToSize);
- buffer[oldSize+linkedToSize] = '\0';
-}
-#endif
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**)
-// Purpose: Private function, read the extended attributes of the file into the block
-// Created: 2005/06/12
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename)
-{
-#ifdef HAVE_SYS_XATTR_H
- int listBufferSize = 10000;
- char* list = new char[listBufferSize];
-
- try
- {
- // This returns an unordered list of attribute names, each 0 terminated,
- // concatenated together
- int listSize = ::llistxattr(Filename, list, listBufferSize);
-
- if(listSize>listBufferSize)
- {
- delete[] list, list = NULL;
- list = new char[listSize];
- listSize = ::llistxattr(Filename, list, listSize);
- }
-
- if(listSize>0)
- {
- // Extract list of attribute names so we can sort them
- std::vector<std::string> attrKeys;
- for(int i = 0; i<listSize; ++i)
- {
- std::string attrKey(list+i);
- i += attrKey.size();
- attrKeys.push_back(attrKey);
- }
- sort(attrKeys.begin(), attrKeys.end());
-
- // Make initial space in block
- int xattrSize = outputBlock.GetSize();
- int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000;
- outputBlock.ResizeBlock(xattrBufferSize);
- unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
-
- // Leave space for attr block size later
- int xattrBlockSizeOffset = xattrSize;
- xattrSize += sizeof(u_int32_t);
-
- // Loop for each attribute
- for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI)
- {
- std::string attrKey(*attrKeyI);
-
- if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize))
- {
- xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2;
- outputBlock.ResizeBlock(xattrBufferSize);
- buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
- }
-
- // Store length and text for attibute name
- u_int16_t keyLength = htons(attrKey.size()+1);
- std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t));
- xattrSize += sizeof(u_int16_t);
- std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1);
- xattrSize += attrKey.size()+1;
-
- // Leave space for value size
- int valueSizeOffset = xattrSize;
- xattrSize += sizeof(u_int32_t);
-
- // Find size of attribute (must call with buffer and length 0 on some platforms,
- // as -1 is returned if the data doesn't fit.)
- int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0);
- if(valueSize<0)
- {
- BOX_LOG_SYS_ERROR("Failed to get "
- "extended attribute size of "
- "'" << Filename << "': " <<
- attrKey);
- THROW_EXCEPTION(CommonException, OSFileError);
- }
-
- // Resize block, if needed
- if(xattrSize+valueSize>xattrBufferSize)
- {
- xattrBufferSize = (xattrBufferSize+valueSize)*2;
- outputBlock.ResizeBlock(xattrBufferSize);
- buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
- }
-
- // This gets the attribute value (may be text or binary), no termination
- valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize);
- if(valueSize<0)
- {
- BOX_LOG_SYS_ERROR("Failed to get "
- "extended attribute of "
- "'" << Filename << "': " <<
- attrKey);
- THROW_EXCEPTION(CommonException, OSFileError);
- }
- xattrSize += valueSize;
-
- // Fill in value size
- u_int32_t valueLength = htonl(valueSize);
- std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t));
- }
-
- // Fill in attribute block size
- u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t));
- std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t));
-
- outputBlock.ResizeBlock(xattrSize);
- }
- else if(listSize<0)
- {
- if(errno == EOPNOTSUPP || errno == EACCES)
- {
- // fail silently
- }
- else if(errno == ERANGE)
- {
- BOX_ERROR("Failed to list extended "
- "attributes of '" << Filename << "': "
- "buffer too small, not backed up");
- }
- else
- {
- BOX_LOG_SYS_ERROR("Failed to list extended "
- "attributes of '" << Filename << "', "
- "not backed up");
- THROW_EXCEPTION(CommonException, OSFileError);
- }
- }
- }
- catch(...)
- {
- delete[] list;
- throw;
- }
- delete[] list;
-#endif
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::GetModificationTimes()
-// Purpose: Returns the modification time embedded in the
-// attributes.
-// Created: 2010/02/24
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::GetModificationTimes(
- box_time_t *pModificationTime,
- box_time_t *pAttrModificationTime) const
-{
- // Got something loaded
- if(GetSize() <= 0)
- {
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
- // Make sure there are clear attributes to use
- EnsureClearAvailable();
- ASSERT(mpClearAttributes != 0);
-
- // Check if the decrypted attributes are small enough, and the type of attributes stored
- if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
- {
- THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
- }
- int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
- ASSERT(type != 0);
- if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
- {
- // Don't know what to do with these
- THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
- }
-
- // Check there is enough space for an attributes block
- if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
- {
- // Too small
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
- // Get pointer to structure
- attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
-
- if(pModificationTime)
- {
- *pModificationTime = box_ntoh64(pattr->ModificationTime);
- }
-
- if(pAttrModificationTime)
- {
- *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime);
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::WriteAttributes(const char *)
-// Purpose: Apply the stored attributes to the file
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::WriteAttributes(const char *Filename,
- bool MakeUserWritable) const
-{
- // Got something loaded
- if(GetSize() <= 0)
- {
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
- // Make sure there are clear attributes to use
- EnsureClearAvailable();
- ASSERT(mpClearAttributes != 0);
-
- // Check if the decrypted attributes are small enough, and the type of attributes stored
- if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
- {
- THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
- }
- int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
- ASSERT(type != 0);
- if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
- {
- // Don't know what to do with these
- THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
- }
-
- // Check there is enough space for an attributes block
- if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
- {
- // Too small
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
- // Get pointer to structure
- attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
- int xattrOffset = sizeof(attr_StreamFormat);
-
- // is it a symlink?
- int16_t mode = ntohs(pattr->Mode);
- if((mode & S_IFMT) == S_IFLNK)
- {
- // Check things are sensible
- if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1)
- {
- // Too small
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
-#ifdef WIN32
- BOX_WARNING("Cannot create symbolic links on Windows: '" <<
- Filename << "'");
-#else
- // Make a symlink, first deleting anything in the way
- ::unlink(Filename);
- if(::symlink((char*)(pattr + 1), Filename) != 0)
- {
- BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename <<
- "' to '" << (char*)(pattr + 1) << "'");
- THROW_EXCEPTION(CommonException, OSFileError)
- }
-#endif
-
- xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1;
- }
-
- // If working as root, set user IDs
- if(::geteuid() == 0)
- {
- #ifndef HAVE_LCHOWN
- // only if not a link, can't set their owner on this platform
- if((mode & S_IFMT) != S_IFLNK)
- {
- // Not a link, use normal chown
- if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
- {
- BOX_LOG_SYS_ERROR("Failed to change "
- "owner of file "
- "'" << Filename << "'");
- THROW_EXCEPTION(CommonException, OSFileError)
- }
- }
- #else
- // use the version which sets things on symlinks
- if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
- {
- BOX_LOG_SYS_ERROR("Failed to change owner of "
- "symbolic link '" << Filename << "'");
- THROW_EXCEPTION(CommonException, OSFileError)
- }
- #endif
- }
-
- if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize())
- {
- WriteExtendedAttr(Filename, xattrOffset);
- }
-
- // Stop now if symlink, because otherwise it'll just be applied to the target
- if((mode & S_IFMT) == S_IFLNK)
- {
- return;
- }
-
- // Set modification time?
- box_time_t modtime = box_ntoh64(pattr->ModificationTime);
- if(modtime != 0)
- {
- // Work out times as timevals
- struct timeval times[2];
-
- #ifdef WIN32
- BoxTimeToTimeval(box_ntoh64(pattr->ModificationTime),
- times[1]);
- BoxTimeToTimeval(box_ntoh64(pattr->AttrModificationTime),
- times[0]);
- // Because stat() returns the creation time in the ctime
- // field under Windows, and this gets saved in the
- // AttrModificationTime field of the serialised attributes,
- // we subvert the first parameter of emu_utimes() to allow
- // it to be reset to the right value on the restored file.
- #else
- BoxTimeToTimeval(modtime, times[1]);
- // Copy access time as well, why not, got to set it to something
- times[0] = times[1];
- // Attr modification time will be changed anyway,
- // nothing that can be done about it
- #endif
-
- // Try to apply
- if(::utimes(Filename, times) != 0)
- {
- BOX_LOG_SYS_WARNING("Failed to change times of "
- "file '" << Filename << "' to ctime=" <<
- BOX_FORMAT_TIMESPEC(times[0]) << ", mtime=" <<
- BOX_FORMAT_TIMESPEC(times[1]));
- }
- }
-
- if (MakeUserWritable)
- {
- mode |= S_IRWXU;
- }
-
- // Apply everything else... (allowable mode flags only)
- // Mode must be done last (think setuid)
- if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID
- | S_ISGID | S_ISVTX)) != 0)
- {
- BOX_LOG_SYS_ERROR("Failed to change permissions of file "
- "'" << Filename << "'");
- THROW_EXCEPTION(CommonException, OSFileError)
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::IsSymLink()
-// Purpose: Do these attributes represent a symbolic link?
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-bool BackupClientFileAttributes::IsSymLink() const
-{
- EnsureClearAvailable();
-
- // Got the right kind of thing?
- if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
- {
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
- // Get the type of attributes stored
- int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
- ASSERT(type != 0);
- if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat))
- {
- // Check link
- attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
- return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK;
- }
-
- return false;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::RemoveClear()
-// Purpose: Private. Deletes any clear version of the attributes that may be held
-// Created: 3/12/03
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::RemoveClear() const
-{
- if(mpClearAttributes)
- {
- delete mpClearAttributes;
- }
- mpClearAttributes = 0;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::EnsureClearAvailable()
-// Purpose: Private. Makes sure the clear version is available
-// Created: 3/12/03
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::EnsureClearAvailable() const
-{
- if(mpClearAttributes == 0)
- {
- mpClearAttributes = MakeClear(*this);
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset)
-// Purpose: Private function, apply the stored extended attributes to the file
-// Created: 2005/06/13
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const
-{
-#ifdef HAVE_SYS_XATTR_H
- const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer());
-
- u_int32_t xattrBlockLength = 0;
- std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t));
- int xattrBlockSize = ntohl(xattrBlockLength);
- xattrOffset += sizeof(u_int32_t);
-
- int xattrEnd = xattrOffset+xattrBlockSize;
- if(xattrEnd>mpClearAttributes->GetSize())
- {
- // Too small
- THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
- }
-
- while(xattrOffset<xattrEnd)
- {
- u_int16_t keyLength = 0;
- std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t));
- int keySize = ntohs(keyLength);
- xattrOffset += sizeof(u_int16_t);
-
- const char* key = buffer+xattrOffset;
- xattrOffset += keySize;
-
- u_int32_t valueLength = 0;
- std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t));
- int valueSize = ntohl(valueLength);
- xattrOffset += sizeof(u_int32_t);
-
- // FIXME: Warn on EOPNOTSUPP
- if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP)
- {
- BOX_LOG_SYS_ERROR("Failed to set extended attributes "
- "on file '" << Filename << "'");
- THROW_EXCEPTION(CommonException, OSFileError);
- }
-
- xattrOffset += valueSize;
- }
-
- ASSERT(xattrOffset==xattrEnd);
-#endif
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &)
-// Purpose: Static. Decrypts stored attributes.
-// Created: 3/12/03
-//
-// --------------------------------------------------------------------------
-StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted)
-{
- // New block
- StreamableMemBlock *pdecrypted = 0;
-
- try
- {
- // Check the block is big enough for IV and header
- int ivSize = sBlowfishEncrypt.GetIVLength();
- if(rEncrypted.GetSize() <= (ivSize + 1))
- {
- THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes);
- }
-
- // How much space is needed for the output?
- int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize);
-
- // Allocate it
- pdecrypted = new StreamableMemBlock(maxDecryptedSize);
-
- // ptr to block
- uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer();
-
- // Check that the header has right type
- if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH)
- {
- THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding);
- }
-
- // Set IV
- sBlowfishDecrypt.SetIV(encBlock + 1);
-
- // Decrypt
- int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1));
-
- // Resize block to fit
- pdecrypted->ResizeBlock(decryptedSize);
- }
- catch(...)
- {
- delete pdecrypted;
- pdecrypted = 0;
- }
-
- return pdecrypted;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int)
-// Purpose: Static. Sets the key to use for encryption and decryption.
-// Created: 3/12/03
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength)
-{
- // IVs set later
- sBlowfishEncrypt.Reset();
- sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
- sBlowfishDecrypt.Reset();
- sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &)
-// Purpose: Private. Encrypt the given attributes into this block.
-// Created: 3/12/03
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt)
-{
- // Free any existing block
- FreeBlock();
-
- // Work out the maximum amount of space we need
- int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize());
- // And the size of the IV
- int ivSize = sBlowfishEncrypt.GetIVLength();
-
- // Allocate this space
- AllocateBlock(maxEncryptedSize + ivSize + 1);
-
- // Store the encoding byte
- uint8_t *block = (uint8_t*)GetBuffer();
- block[0] = ATTRIBUTE_ENCODING_BLOWFISH;
-
- // Generate and store an IV for this attribute block
- int ivSize2 = 0;
- const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2);
- ASSERT(ivSize == ivSize2);
-
- // Copy into the encrypted block
- ::memcpy(block + 1, iv, ivSize);
-
- // Do the transform
- int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize());
-
- // Resize this block
- ResizeBlock(encrytedSize + ivSize + 1);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int)
-// Purpose: Set the secret for the filename attribute hash
-// Created: 25/4/04
-//
-// --------------------------------------------------------------------------
-void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength)
-{
- if(SecretLength > (int)sizeof(sAttributeHashSecret))
- {
- SecretLength = sizeof(sAttributeHashSecret);
- }
- if(SecretLength < 0)
- {
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-
- // Copy
- ::memcpy(sAttributeHashSecret, pSecret, SecretLength);
- sAttributeHashSecretLength = SecretLength;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupClientFileAttributes::GenerateAttributeHash(
-// struct stat &, const std::string &,
-// const std::string &)
-// Purpose: Generate a 64 bit hash from the attributes, used to
-// detect changes. Include filename in the hash, so
-// that it changes from one file to another, so don't
-// reveal identical attributes.
-// Created: 25/4/04
-//
-// --------------------------------------------------------------------------
-uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st,
- const std::string &filename, const std::string &leafname)
-{
- if(sAttributeHashSecretLength == 0)
- {
- THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet)
- }
-
- // Assemble stuff we're interested in
- attributeHashData hashData;
- memset(&hashData, 0, sizeof(hashData));
- // Use network byte order and large sizes to be cross platform
- hashData.uid = htonl(st.st_uid);
- hashData.gid = htonl(st.st_gid);
- hashData.mode = htonl(st.st_mode);
-
- #ifdef WIN32
- // On Windows, the "file attribute modification time" is the
- // file creation time, and we want to back this up, restore
- // it and compare it.
- //
- // On other platforms, it's not very important and can't
- // reliably be set to anything other than the current time.
- hashData.fileCreationTime = box_hton64(st.st_ctime);
- #endif
-
- StreamableMemBlock xattr;
- FillExtendedAttr(xattr, filename.c_str());
-
- // Create a MD5 hash of the data, filename, and secret
- MD5Digest digest;
- digest.Add(&hashData, sizeof(hashData));
- digest.Add(xattr.GetBuffer(), xattr.GetSize());
- digest.Add(leafname.c_str(), leafname.size());
- digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
- digest.Finish();
-
- // Return the first 64 bits of the hash
- uint64_t result = *((uint64_t *)(digest.DigestAsData()));
- return result;
-}
Deleted: box/trunk/lib/backupclient/BackupClientFileAttributes.h
===================================================================
--- box/trunk/lib/backupclient/BackupClientFileAttributes.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupClientFileAttributes.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,78 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupClientFileAttributes.h
-// Purpose: Storage of file attributes
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPCLIENTFILEATTRIBUTES__H
-#define BACKUPCLIENTFILEATTRIBUTES__H
-
-#include <string>
-
-#include "StreamableMemBlock.h"
-#include "BoxTime.h"
-
-EMU_STRUCT_STAT; // declaration
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupClientFileAttributes
-// Purpose: Storage, streaming and application of file attributes
-// Created: 2003/10/07
-//
-// --------------------------------------------------------------------------
-class BackupClientFileAttributes : public StreamableMemBlock
-{
-public:
- BackupClientFileAttributes();
- BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy);
- BackupClientFileAttributes(const StreamableMemBlock &rToCopy);
- ~BackupClientFileAttributes();
- BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr);
- BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr);
- bool operator==(const BackupClientFileAttributes &rAttr) const;
-// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous?
-
- bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const;
-
- // Prevent access to base class members accidently
- void Set();
-
- void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false,
- box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0,
- InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0);
- void WriteAttributes(const char *Filename,
- bool MakeUserWritable = false) const;
- void GetModificationTimes(box_time_t *pModificationTime,
- box_time_t *pAttrModificationTime) const;
-
- bool IsSymLink() const;
-
- static void SetBlowfishKey(const void *pKey, int KeyLength);
- static void SetAttributeHashSecret(const void *pSecret, int SecretLength);
-
- static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname);
- static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename);
-
-private:
- static void FillAttributes(StreamableMemBlock &outputBlock,
- const char *Filename, EMU_STRUCT_STAT &st,
- bool ZeroModificationTimes);
- static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st);
- void WriteExtendedAttr(const char *Filename, int xattrOffset) const;
-
- void RemoveClear() const;
- void EnsureClearAvailable() const;
- static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted);
- void EncryptAttr(const StreamableMemBlock &rToEncrypt);
-
-private:
- mutable StreamableMemBlock *mpClearAttributes;
-};
-
-#endif // BACKUPCLIENTFILEATTRIBUTES__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreConstants.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreConstants.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreConstants.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,44 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreContants.h
-// Purpose: constants for the backup system
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTORECONSTANTS__H
-#define BACKUPSTORECONSTANTS__H
-
-#define BACKUPSTORE_ROOT_DIRECTORY_ID 1
-
-#define BACKUP_STORE_SERVER_VERSION 1
-
-// Minimum size for a chunk to be compressed
-#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256
-
-// min and max sizes for blocks
-#define BACKUP_FILE_MIN_BLOCK_SIZE 4096
-#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024)
-
-// Increase the block size if there are more than this number of blocks
-#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096
-
-// Avoid creating blocks smaller than this
-#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128
-
-// Maximum number of sizes to do an rsync-like scan for
-#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64
-
-// When doing rsync scans, do not scan for blocks smaller than
-#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128
-
-// A limit to stop diffing running out of control: If more than this
-// times the number of blocks in the original index are found, stop
-// looking. This stops really bad cases of diffing files containing
-// all the same byte using huge amounts of memory and processor time.
-// This is a multiple of the number of blocks in the diff from file.
-#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096
-
-#endif // BACKUPSTORECONSTANTS__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreDirectory.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreDirectory.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreDirectory.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,568 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreDirectory.h
-// Purpose: Representation of a backup directory
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include <sys/types.h>
-
-#include "BackupStoreDirectory.h"
-#include "IOStream.h"
-#include "BackupStoreException.h"
-#include "BackupStoreObjectMagic.h"
-
-#include "MemLeakFindOn.h"
-
-// set packing to one byte
-#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
-#include "BeginStructPackForWire.h"
-#else
-BEGIN_STRUCTURE_PACKING_FOR_WIRE
-#endif
-
-typedef struct
-{
- int32_t mMagicValue; // also the version number
- int32_t mNumEntries;
- int64_t mObjectID; // this object ID
- int64_t mContainerID; // ID of container
- uint64_t mAttributesModTime;
- int32_t mOptionsPresent; // bit mask of optional sections / features present
- // Then a StreamableMemBlock for attributes
-} dir_StreamFormat;
-
-typedef enum
-{
- Option_DependencyInfoPresent = 1
-} dir_StreamFormatOptions;
-
-typedef struct
-{
- uint64_t mModificationTime;
- int64_t mObjectID;
- int64_t mSizeInBlocks;
- uint64_t mAttributesHash;
- int16_t mFlags; // order smaller items after bigger ones (for alignment)
- // Then a BackupStoreFilename
- // Then a StreamableMemBlock for attributes
-} en_StreamFormat;
-
-typedef struct
-{
- int64_t mDependsNewer;
- int64_t mDependsOlder;
-} en_StreamFormatDepends;
-
-// Use default packing
-#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
-#include "EndStructPackForWire.h"
-#else
-END_STRUCTURE_PACKING_FOR_WIRE
-#endif
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::BackupStoreDirectory()
-// Purpose: Constructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::BackupStoreDirectory()
- : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0)
-{
- ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t)
-// Purpose: Constructor giving object and container IDs
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID)
- : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0)
-{
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::~BackupStoreDirectory()
-// Purpose: Destructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::~BackupStoreDirectory()
-{
- for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
- {
- delete (*i);
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int)
-// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads.
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout)
-{
- // Get the header
- dir_StreamFormat hdr;
- if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Check magic value...
- if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue))
- {
- THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat)
- }
-
- // Get data
- mObjectID = box_ntoh64(hdr.mObjectID);
- mContainerID = box_ntoh64(hdr.mContainerID);
- mAttributesModTime = box_ntoh64(hdr.mAttributesModTime);
-
- // Options
- int32_t options = ntohl(hdr.mOptionsPresent);
-
- // Get attributes
- mAttributes.ReadFromStream(rStream, Timeout);
-
- // Decode count
- int count = ntohl(hdr.mNumEntries);
-
- // Clear existing list
- for(std::vector<Entry*>::iterator i = mEntries.begin();
- i != mEntries.end(); i++)
- {
- delete (*i);
- }
- mEntries.clear();
-
- // Read them in!
- for(int c = 0; c < count; ++c)
- {
- Entry *pen = new Entry;
- try
- {
- // Read from stream
- pen->ReadFromStream(rStream, Timeout);
-
- // Add to list
- mEntries.push_back(pen);
- }
- catch(...)
- {
- delete pen;
- throw;
- }
- }
-
- // Read in dependency info?
- if(options & Option_DependencyInfoPresent)
- {
- // Read in extra dependency data
- for(int c = 0; c < count; ++c)
- {
- mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout);
- }
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool)
-// Purpose: Writes a selection of entries to a stream
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const
-{
- // Get count of entries
- int32_t count = mEntries.size();
- if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING)
- {
- // Need to count the entries
- count = 0;
- Iterator i(*this);
- while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0)
- {
- count++;
- }
- }
-
- // Check that sensible IDs have been set
- ASSERT(mObjectID != 0);
- ASSERT(mContainerID != 0);
-
- // Need dependency info?
- bool dependencyInfoRequired = false;
- if(StreamDependencyInfo)
- {
- Iterator i(*this);
- Entry *pen = 0;
- while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
- {
- if(pen->HasDependencies())
- {
- dependencyInfoRequired = true;
- }
- }
- }
-
- // Options
- int32_t options = 0;
- if(dependencyInfoRequired) options |= Option_DependencyInfoPresent;
-
- // Build header
- dir_StreamFormat hdr;
- hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE);
- hdr.mNumEntries = htonl(count);
- hdr.mObjectID = box_hton64(mObjectID);
- hdr.mContainerID = box_hton64(mContainerID);
- hdr.mAttributesModTime = box_hton64(mAttributesModTime);
- hdr.mOptionsPresent = htonl(options);
-
- // Write header
- rStream.Write(&hdr, sizeof(hdr));
-
- // Write the attributes?
- if(StreamAttributes)
- {
- mAttributes.WriteToStream(rStream);
- }
- else
- {
- // Write a blank header instead
- StreamableMemBlock::WriteEmptyBlockToStream(rStream);
- }
-
- // Then write all the entries
- Iterator i(*this);
- Entry *pen = 0;
- while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
- {
- pen->WriteToStream(rStream);
- }
-
- // Write dependency info?
- if(dependencyInfoRequired)
- {
- Iterator i(*this);
- Entry *pen = 0;
- while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
- {
- pen->WriteToStreamDependencyInfo(rStream);
- }
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::AddEntry(const Entry &)
-// Purpose: Adds entry to directory (no checking)
-// Created: 2003/08/27
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy)
-{
- Entry *pnew = new Entry(rEntryToCopy);
- try
- {
- mEntries.push_back(pnew);
- }
- catch(...)
- {
- delete pnew;
- throw;
- }
-
- return pnew;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
-// Purpose: Adds entry to directory (no checking)
-// Created: 2003/08/27
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime)
-{
- Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime);
- try
- {
- mEntries.push_back(pnew);
- }
- catch(...)
- {
- delete pnew;
- throw;
- }
-
- return pnew;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::DeleteEntry(int64_t)
-// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient)
-// Created: 2003/08/27
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::DeleteEntry(int64_t ObjectID)
-{
- for(std::vector<Entry*>::iterator i(mEntries.begin());
- i != mEntries.end(); ++i)
- {
- if((*i)->mObjectID == ObjectID)
- {
- // Delete
- delete (*i);
- // Remove from list
- mEntries.erase(i);
- // Done
- return;
- }
- }
-
- // Not found
- THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::FindEntryByID(int64_t)
-// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist.
-// Created: 12/11/03
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const
-{
- for(std::vector<Entry*>::const_iterator i(mEntries.begin());
- i != mEntries.end(); ++i)
- {
- if((*i)->mObjectID == ObjectID)
- {
- // Found
- return (*i);
- }
- }
-
- // Not found
- return 0;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::Entry()
-// Purpose: Constructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry::Entry()
- : mModificationTime(0),
- mObjectID(0),
- mSizeInBlocks(0),
- mFlags(0),
- mAttributesHash(0),
- mMinMarkNumber(0),
- mMarkNumber(0),
- mDependsNewer(0),
- mDependsOlder(0)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::~Entry()
-// Purpose: Destructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry::~Entry()
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::Entry(const Entry &)
-// Purpose: Copy constructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry::Entry(const Entry &rToCopy)
- : mName(rToCopy.mName),
- mModificationTime(rToCopy.mModificationTime),
- mObjectID(rToCopy.mObjectID),
- mSizeInBlocks(rToCopy.mSizeInBlocks),
- mFlags(rToCopy.mFlags),
- mAttributesHash(rToCopy.mAttributesHash),
- mAttributes(rToCopy.mAttributes),
- mMinMarkNumber(rToCopy.mMinMarkNumber),
- mMarkNumber(rToCopy.mMarkNumber),
- mDependsNewer(rToCopy.mDependsNewer),
- mDependsOlder(rToCopy.mDependsOlder)
-{
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
-// Purpose: Constructor from values
-// Created: 2003/08/27
-//
-// --------------------------------------------------------------------------
-BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash)
- : mName(rName),
- mModificationTime(ModificationTime),
- mObjectID(ObjectID),
- mSizeInBlocks(SizeInBlocks),
- mFlags(Flags),
- mAttributesHash(AttributesHash),
- mMinMarkNumber(0),
- mMarkNumber(0),
- mDependsNewer(0),
- mDependsOlder(0)
-{
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int)
-// Purpose: Read an entry from a stream
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout)
-{
- // Grab the raw bytes from the stream which compose the header
- en_StreamFormat entry;
- if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Do reading first before modifying the variables, to be more exception safe
-
- // Get the filename
- BackupStoreFilename name;
- name.ReadFromStream(rStream, Timeout);
-
- // Get the attributes
- mAttributes.ReadFromStream(rStream, Timeout);
-
- // Store the rest of the bits
- mModificationTime = box_ntoh64(entry.mModificationTime);
- mObjectID = box_ntoh64(entry.mObjectID);
- mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks);
- mAttributesHash = box_ntoh64(entry.mAttributesHash);
- mFlags = ntohs(entry.mFlags);
- mName = name;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &)
-// Purpose: Writes the entry to a stream
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const
-{
- // Build a structure
- en_StreamFormat entry;
- entry.mModificationTime = box_hton64(mModificationTime);
- entry.mObjectID = box_hton64(mObjectID);
- entry.mSizeInBlocks = box_hton64(mSizeInBlocks);
- entry.mAttributesHash = box_hton64(mAttributesHash);
- entry.mFlags = htons(mFlags);
-
- // Write it
- rStream.Write(&entry, sizeof(entry));
-
- // Write the filename
- mName.WriteToStream(rStream);
-
- // Write any attributes
- mAttributes.WriteToStream(rStream);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int)
-// Purpose: Read the optional dependency info from a stream
-// Created: 13/7/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout)
-{
- // Grab the raw bytes from the stream which compose the header
- en_StreamFormatDepends depends;
- if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Store the data
- mDependsNewer = box_ntoh64(depends.mDependsNewer);
- mDependsOlder = box_ntoh64(depends.mDependsOlder);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &)
-// Purpose: Write the optional dependency info to a stream
-// Created: 13/7/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const
-{
- // Build structure
- en_StreamFormatDepends depends;
- depends.mDependsNewer = box_hton64(mDependsNewer);
- depends.mDependsOlder = box_hton64(mDependsOlder);
- // Write
- rStream.Write(&depends, sizeof(depends));
-}
-
-
-
Deleted: box/trunk/lib/backupclient/BackupStoreDirectory.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreDirectory.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreDirectory.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,285 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreDirectory.h
-// Purpose: Representation of a backup directory
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREDIRECTORY__H
-#define BACKUPSTOREDIRECTORY__H
-
-#include <string>
-#include <vector>
-
-#include "BackupStoreFilenameClear.h"
-#include "StreamableMemBlock.h"
-#include "BoxTime.h"
-
-class IOStream;
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupStoreDirectory
-// Purpose: In memory representation of a directory
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-class BackupStoreDirectory
-{
-public:
- BackupStoreDirectory();
- BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID);
-private:
- // Copying not allowed
- BackupStoreDirectory(const BackupStoreDirectory &rToCopy);
-public:
- ~BackupStoreDirectory();
-
- class Entry
- {
- public:
- friend class BackupStoreDirectory;
-
- Entry();
- ~Entry();
- Entry(const Entry &rToCopy);
- Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash);
-
- void ReadFromStream(IOStream &rStream, int Timeout);
- void WriteToStream(IOStream &rStream) const;
-
- const BackupStoreFilename &GetName() const {return mName;}
- box_time_t GetModificationTime() const {return mModificationTime;}
- int64_t GetObjectID() const {return mObjectID;}
- int64_t GetSizeInBlocks() const {return mSizeInBlocks;}
- int16_t GetFlags() const {return mFlags;}
- void AddFlags(int16_t Flags) {mFlags |= Flags;}
- void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;}
-
- // Some things can be changed
- void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;}
- void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;}
-
- // Attributes
- bool HasAttributes() const {return !mAttributes.IsEmpty();}
- void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;}
- const StreamableMemBlock &GetAttributes() const {return mAttributes;}
- uint64_t GetAttributesHash() const {return mAttributesHash;}
-
- // Marks
- // The lowest mark number a version of a file of this name has ever had
- uint32_t GetMinMarkNumber() const {return mMinMarkNumber;}
- // The mark number on this file
- uint32_t GetMarkNumber() const {return mMarkNumber;}
-
- // Make sure these flags are synced with those in backupprocotol.txt
- // ListDirectory command
- enum
- {
- Flags_INCLUDE_EVERYTHING = -1,
- Flags_EXCLUDE_NOTHING = 0,
- Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below!
- Flags_File = 1,
- Flags_Dir = 2,
- Flags_Deleted = 4,
- Flags_OldVersion = 8,
- Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion
- };
- // characters for textual listing of files -- see bbackupquery/BackupQueries
- #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR"
-
- // convenience methods
- bool inline IsDir()
- {
- return GetFlags() & Flags_Dir;
- }
- bool inline IsFile()
- {
- return GetFlags() & Flags_File;
- }
- bool inline IsOld()
- {
- return GetFlags() & Flags_OldVersion;
- }
- bool inline IsDeleted()
- {
- return GetFlags() & Flags_Deleted;
- }
- bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
- {
- return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet))
- && ((mFlags & FlagsNotToBeSet) == 0);
- };
-
- // Get dependency info
- // new version this depends on
- int64_t GetDependsNewer() const {return mDependsNewer;}
- void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;}
- // older version which depends on this
- int64_t GetDependsOlder() const {return mDependsOlder;}
- void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;}
-
- // Dependency info saving
- bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;}
- void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout);
- void WriteToStreamDependencyInfo(IOStream &rStream) const;
-
- private:
- BackupStoreFilename mName;
- box_time_t mModificationTime;
- int64_t mObjectID;
- int64_t mSizeInBlocks;
- int16_t mFlags;
- uint64_t mAttributesHash;
- StreamableMemBlock mAttributes;
- uint32_t mMinMarkNumber;
- uint32_t mMarkNumber;
-
- uint64_t mDependsNewer; // new version this depends on
- uint64_t mDependsOlder; // older version which depends on this
- };
-
- void ReadFromStream(IOStream &rStream, int Timeout);
- void WriteToStream(IOStream &rStream,
- int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING,
- int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING,
- bool StreamAttributes = true, bool StreamDependencyInfo = true) const;
-
- Entry *AddEntry(const Entry &rEntryToCopy);
- Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime);
- void DeleteEntry(int64_t ObjectID);
- Entry *FindEntryByID(int64_t ObjectID) const;
-
- int64_t GetObjectID() const {return mObjectID;}
- int64_t GetContainerID() const {return mContainerID;}
-
- // Need to be able to update the container ID when moving objects
- void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;}
-
- // Purely for use of server -- not serialised into streams
- int64_t GetRevisionID() const {return mRevisionID;}
- void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;}
-
- unsigned int GetNumberOfEntries() const {return mEntries.size();}
-
- // User info -- not serialised into streams
- int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;}
- void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;}
-
- // Attributes
- bool HasAttributes() const {return !mAttributes.IsEmpty();}
- void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;}
- const StreamableMemBlock &GetAttributes() const {return mAttributes;}
- box_time_t GetAttributesModTime() const {return mAttributesModTime;}
-
- class Iterator
- {
- public:
- Iterator(const BackupStoreDirectory &rDir)
- : mrDir(rDir), i(rDir.mEntries.begin())
- {
- }
-
- BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
- {
- // Skip over things which don't match the required flags
- while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
- {
- ++i;
- }
- // Not the last one?
- if(i == mrDir.mEntries.end())
- {
- return 0;
- }
- // Return entry, and increment
- return (*(i++));
- }
-
- // WARNING: This function is really very inefficient.
- // Only use when you want to look up ONE filename, not in a loop looking up lots.
- // In a looping situation, cache the decrypted filenames in another memory structure.
- BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
- {
- // Skip over things which don't match the required flags or filename
- while( (i != mrDir.mEntries.end())
- && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
- || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) )
- {
- ++i;
- }
- // Not the last one?
- if(i == mrDir.mEntries.end())
- {
- return 0;
- }
- // Return entry, and increment
- return (*(i++));
- }
-
- private:
- const BackupStoreDirectory &mrDir;
- std::vector<Entry*>::const_iterator i;
- };
-
- friend class Iterator;
-
- class ReverseIterator
- {
- public:
- ReverseIterator(const BackupStoreDirectory &rDir)
- : mrDir(rDir), i(rDir.mEntries.rbegin())
- {
- }
-
- BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
- {
- // Skip over things which don't match the required flags
- while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
- {
- ++i;
- }
- // Not the last one?
- if(i == mrDir.mEntries.rend())
- {
- return 0;
- }
- // Return entry, and increment
- return (*(i++));
- }
-
- private:
- const BackupStoreDirectory &mrDir;
- std::vector<Entry*>::const_reverse_iterator i;
- };
-
- friend class ReverseIterator;
-
- // For recovery of the store
- // Implemented in BackupStoreCheck2.cpp
- bool CheckAndFix();
- void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags);
- bool NameInUse(const BackupStoreFilename &rName);
- // Don't use these functions in normal code!
-
- // For testing
- void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;}
-
- // Debug and diagonistics
- void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere
-
-private:
- int64_t mRevisionID;
- int64_t mObjectID;
- int64_t mContainerID;
- std::vector<Entry*> mEntries;
- box_time_t mAttributesModTime;
- StreamableMemBlock mAttributes;
- int64_t mUserInfo1;
-};
-
-#endif // BACKUPSTOREDIRECTORY__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreException.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreException.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreException.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,17 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreException.h
-// Purpose: Exception
-// Created: 2003/07/08
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREEXCEPTION__H
-#define BACKUPSTOREEXCEPTION__H
-
-// Compatibility
-#include "autogen_BackupStoreException.h"
-
-#endif // BACKUPSTOREEXCEPTION__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreException.txt
===================================================================
--- box/trunk/lib/backupclient/BackupStoreException.txt 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreException.txt 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,72 +0,0 @@
-EXCEPTION BackupStore 4
-
-Internal 0
-BadAccountDatabaseFile 1
-AccountDatabaseNoSuchEntry 2
-InvalidBackupStoreFilename 3
-UnknownFilenameEncoding 4
-CouldntReadEntireStructureFromStream 5
-BadDirectoryFormat 6
-CouldNotFindEntryInDirectory 7
-OutputFileAlreadyExists 8
-OSFileError 9
-StreamDoesntHaveRequiredFeatures 10
-BadBackupStoreFile 11
-CouldNotLoadStoreInfo 12
-BadStoreInfoOnLoad 13
-StoreInfoIsReadOnly 14
-StoreInfoDirNotInList 15
-StoreInfoBlockDeltaMakesValueNegative 16
-DirectoryHasBeenDeleted 17
-StoreInfoNotInitialised 18
-StoreInfoAlreadyLoaded 19
-StoreInfoNotLoaded 20
-ReadFileFromStreamTimedOut 21
-FileWrongSizeAfterBeingStored 22
-AddedFileDoesNotVerify 23
-StoreInfoForWrongAccount 24
-ContextIsReadOnly 25
-AttributesNotLoaded 26
-AttributesNotUnderstood 27
-WrongServerVersion 28 # client side
-ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account.
-NameAlreadyExistsInDirectory 30
-BerkelyDBFailure 31 # client side
-InodeMapIsReadOnly 32 # client side
-InodeMapNotOpen 33 # client side
-FilenameEncryptionKeyNotKnown 34
-FilenameEncryptionNoKeyForSpecifiedMethod 35
-FilenameEncryptionNotSetup 36
-CouldntLoadClientKeyMaterial 37
-BadEncryptedAttributes 38
-EncryptedAttributesHaveUnknownEncoding 39
-OutputSizeTooSmallForChunk 40
-BadEncodedChunk 41
-NotEnoughSpaceToDecodeChunk 42
-ChunkHasUnknownEncoding 43
-ChunkContainsBadCompressedData 44
-CantWriteToEncodedFileStream 45
-Temp_FileEncodeStreamDidntReadBuffer 46
-CantWriteToDecodedFileStream 47
-WhenDecodingExpectedToReadButCouldnt 48
-BackupStoreFileFailedIntegrityCheck 49
-ThereIsNoDataInASymLink 50
-IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51
-BlockEntryEncodingDidntGiveExpectedLength 52
-CouldNotFindUnusedIDDuringAllocation 53
-AddedFileExceedsStorageLimit 54
-CannotDiffAnIncompleteStoreFile 55
-CannotDecodeDiffedFilesWithoutCombining 56
-FailedToReadBlockOnCombine 57
-OnCombineFromFileIsIncomplete 58
-BadNotifySysadminEventCode 59
-InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60
-CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server?
-AttributeHashSecretNotSet 62
-AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES
-SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection.
-IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related
-DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory
-PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
-UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known.
-MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first
Deleted: box/trunk/lib/backupclient/BackupStoreFile.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFile.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFile.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,1559 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFile.cpp
-// Purpose: Utils for manipulating files
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#ifdef HAVE_UNISTD_H
- #include <unistd.h>
-#endif
-
-#include <sys/stat.h>
-#include <string.h>
-#include <new>
-#include <string.h>
-
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- #include <stdio.h>
-#endif
-
-#include "BackupStoreFile.h"
-#include "BackupStoreFileWire.h"
-#include "BackupStoreFileCryptVar.h"
-#include "BackupStoreFilename.h"
-#include "BackupStoreException.h"
-#include "IOStream.h"
-#include "Guards.h"
-#include "FileModificationTime.h"
-#include "FileStream.h"
-#include "BackupClientFileAttributes.h"
-#include "BackupStoreObjectMagic.h"
-#include "Compress.h"
-#include "CipherContext.h"
-#include "CipherBlowfish.h"
-#include "CipherAES.h"
-#include "BackupStoreConstants.h"
-#include "CollectInBufferStream.h"
-#include "RollingChecksum.h"
-#include "MD5Digest.h"
-#include "ReadGatherStream.h"
-#include "Random.h"
-#include "BackupStoreFileEncodeStream.h"
-#include "Logging.h"
-
-#include "MemLeakFindOn.h"
-
-using namespace BackupStoreFileCryptVar;
-
-// How big a buffer to use for copying files
-#define COPY_BUFFER_SIZE (8*1024)
-
-// Statistics
-BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
-
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- bool sWarnedAboutBackwardsCompatiblity = false;
-#endif
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &)
-// Purpose: Encode a file into something for storing on file server.
-// Requires a real filename so full info can be stored.
-//
-// Returns a stream. Most of the work is done by the stream
-// when data is actually requested -- the file will be held
-// open until the stream is deleted or the file finished.
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(
- const char *Filename, int64_t ContainerID,
- const BackupStoreFilename &rStoreFilename,
- int64_t *pModificationTime,
- ReadLoggingStream::Logger* pLogger,
- RunStatusProvider* pRunStatusProvider)
-{
- // Create the stream
- std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
-
- // Do the initial setup
- ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename,
- 0 /* no recipe, just encode */,
- ContainerID, rStoreFilename, pModificationTime, pLogger,
- pRunStatusProvider);
-
- // Return the stream for the caller
- return stream;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
-// Purpose: Verify that an encoded file meets the format
-// requirements. Doesn't verify that the data is intact
-// and can be decoded. Optionally returns the ID of the
-// file which it is diffed from, and the (original)
-// container ID.
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
-{
- // Get the size of the file
- int64_t fileSize = rFile.BytesLeftToRead();
- if(fileSize == IOStream::SizeOfStreamUnknown)
- {
- THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
- }
-
- // Get the header...
- file_StreamFormat hdr;
- if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
- {
- // Couldn't read header
- return false;
- }
-
- // Check magic number
- if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
-#endif
- )
- {
- return false;
- }
-
- // Get a filename, see if it loads OK
- try
- {
- BackupStoreFilename fn;
- fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
- }
- catch(...)
- {
- // an error occured while reading it, so that's not good
- return false;
- }
-
- // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
- try
- {
- int32_t size_s;
- if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
- {
- THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
- }
- int size = ntohl(size_s);
- // Skip forward the size
- rFile.Seek(size, IOStream::SeekType_Relative);
- }
- catch(...)
- {
- // an error occured while reading it, so that's not good
- return false;
- }
-
- // Get current position in file -- the end of the header
- int64_t headerEnd = rFile.GetPosition();
-
- // Get number of blocks
- int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
-
- // Calculate where the block index will be, check it's reasonable
- int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
- if(blockIndexLoc < headerEnd)
- {
- // Not enough space left for the block index, let alone the blocks themselves
- return false;
- }
-
- // Load the block index header
- rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
- file_BlockIndexHeader blkhdr;
- if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
- {
- // Couldn't read block index header -- assume bad file
- return false;
- }
-
- // Check header
- if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
-#endif
- )
- || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks)
- {
- // Bad header -- either magic value or number of blocks is wrong
- return false;
- }
-
- // Flag for recording whether a block is referenced from another file
- bool blockFromOtherFileReferenced = false;
-
- // Read the index, checking that the length values all make sense
- int64_t currentBlockStart = headerEnd;
- for(int64_t b = 0; b < numBlocks; ++b)
- {
- // Read block entry
- file_BlockIndexEntry blk;
- if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
- {
- // Couldn't read block index entry -- assume bad file
- return false;
- }
-
- // Check size and location
- int64_t blkSize = box_ntoh64(blk.mEncodedSize);
- if(blkSize <= 0)
- {
- // Mark that this file references another file
- blockFromOtherFileReferenced = true;
- }
- else
- {
- // This block is actually in this file
- if((currentBlockStart + blkSize) > blockIndexLoc)
- {
- // Encoded size makes the block run over the index
- return false;
- }
-
- // Move the current block start ot the end of this block
- currentBlockStart += blkSize;
- }
- }
-
- // Check that there's no empty space
- if(currentBlockStart != blockIndexLoc)
- {
- return false;
- }
-
- // Check that if another block is references, then the ID is there, and if one isn't there is no ID.
- int64_t otherID = box_ntoh64(blkhdr.mOtherFileID);
- if((otherID != 0 && blockFromOtherFileReferenced == false)
- || (otherID == 0 && blockFromOtherFileReferenced == true))
- {
- // Doesn't look good!
- return false;
- }
-
- // Does the caller want the other ID?
- if(pDiffFromObjectIDOut)
- {
- *pDiffFromObjectIDOut = otherID;
- }
-
- // Does the caller want the container ID?
- if(pContainerIDOut)
- {
- *pContainerIDOut = box_ntoh64(hdr.mContainerID);
- }
-
- // Passes all tests
- return true;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodeFile(IOStream &, const char *)
-// Purpose: Decode a file. Will set file attributes. File must not exist.
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
-{
- // Does file exist?
- EMU_STRUCT_STAT st;
- if(EMU_STAT(DecodedFilename, &st) == 0)
- {
- THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
- }
-
- // Try, delete output file if error
- try
- {
- // Make a stream for outputting this file
- FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
-
- // Get the decoding stream
- std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
-
- // Is it a symlink?
- if(!stream->IsSymLink())
- {
- // Copy it out to the file
- stream->CopyStreamTo(out);
- }
-
- out.Close();
-
- // The stream might have uncertain size, in which case
- // we need to drain it to get the
- // Protocol::ProtocolStreamHeader_EndOfStream byte
- // out of our connection stream.
- char buffer[1];
- int drained = rEncodedFile.Read(buffer, 1);
-
- // The Read will return 0 if we are actually at the end
- // of the stream, but some tests decode files directly,
- // in which case we are actually positioned at the start
- // of the block index. I hope that reading an extra byte
- // doesn't hurt!
- // ASSERT(drained == 0);
-
- // Write the attributes
- try
- {
- stream->GetAttributes().WriteAttributes(DecodedFilename);
- }
- catch (std::exception& e)
- {
- BOX_WARNING("Failed to restore attributes on " <<
- DecodedFilename << ": " << e.what());
- }
- }
- catch(...)
- {
- ::unlink(DecodedFilename);
- throw;
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
-// Purpose: Return a stream which will decode the encrypted file data on the fly.
-// Accepts streams in block index first, or main header first, order. In the latter case,
-// the stream must be Seek()able.
-//
-// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
-// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
-{
- // Create stream
- std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
-
- // Get it ready
- stream->Setup(pAlterativeAttr);
-
- // Return to caller
- return stream;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
-// Purpose: Constructor
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
- : mrEncodedFile(rEncodedFile),
- mTimeout(Timeout),
- mNumBlocks(0),
- mpBlockIndex(0),
- mpEncodedData(0),
- mpClearData(0),
- mClearDataSize(0),
- mCurrentBlock(-1),
- mCurrentBlockClearSize(0),
- mPositionInCurrentBlock(0),
- mEntryIVBase(42) // different to default value in the encoded stream!
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- , mIsOldVersion(false)
-#endif
-{
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::~DecodedStream()
-// Purpose: Desctructor
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-BackupStoreFile::DecodedStream::~DecodedStream()
-{
- // Free any allocated memory
- if(mpBlockIndex)
- {
- ::free(mpBlockIndex);
- }
- if(mpEncodedData)
- {
- BackupStoreFile::CodingChunkFree(mpEncodedData);
- }
- if(mpClearData)
- {
- ::free(mpClearData);
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
-// Purpose: Get the stream ready to decode -- reads in headers
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
-{
- // Get the size of the file
- int64_t fileSize = mrEncodedFile.BytesLeftToRead();
-
- // Get the magic number to work out which order the stream is in
- int32_t magic;
- if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read magic value
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
-
- bool inFileOrder = true;
- switch(ntohl(magic))
- {
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
- mIsOldVersion = true;
- // control flows on
-#endif
- case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
- inFileOrder = true;
- break;
-
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
- mIsOldVersion = true;
- // control flows on
-#endif
- case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
- inFileOrder = false;
- break;
-
- default:
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // If not in file order, then the index list must be read now
- if(!inFileOrder)
- {
- ReadBlockIndex(true /* have already read and verified the magic number */);
- }
-
- // Get header
- file_StreamFormat hdr;
- if(inFileOrder)
- {
- // Read the header, without the magic number
- if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
- 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
- // Put in magic number
- hdr.mMagicValue = magic;
- }
- else
- {
- // Not in file order, so need to read the full header
- if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
- }
-
- // Check magic number
- if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
-#endif
- )
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // Get the filename
- mFilename.ReadFromStream(mrEncodedFile, mTimeout);
-
- // Get the attributes (either from stream, or supplied attributes)
- if(pAlterativeAttr != 0)
- {
- // Read dummy attributes
- BackupClientFileAttributes attr;
- attr.ReadFromStream(mrEncodedFile, mTimeout);
-
- // Set to supplied attributes
- mAttributes = *pAlterativeAttr;
- }
- else
- {
- // Read the attributes from the stream
- mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
- }
-
- // If it is in file order, go and read the file attributes
- // Requires that the stream can seek
- if(inFileOrder)
- {
- // Make sure the file size is known
- if(fileSize == IOStream::SizeOfStreamUnknown)
- {
- THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
- }
-
- // Store current location (beginning of encoded blocks)
- int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
-
- // Work out where the index is
- int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
- int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
-
- // Seek to that position
- mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
-
- // Read the block index
- ReadBlockIndex(false /* magic number still to be read */);
-
- // Seek back to the end of header position, ready for reading the chunks
- mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
- }
-
- // Check view of blocks from block header and file header match
- if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks))
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // Need to allocate some memory for the two blocks for reading encoded data, and clear data
- if(mNumBlocks > 0)
- {
- // Find the maximum encoded data size
- int32_t maxEncodedDataSize = 0;
- const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
- ASSERT(entry != 0);
- for(int64_t e = 0; e < mNumBlocks; e++)
- {
- // Get the clear and encoded size
- int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize);
- ASSERT(encodedSize > 0);
-
- // Larger?
- if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
- }
-
- // Allocate those blocks!
- mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
-
- // Allocate the block for the clear data, using the hint from the header.
- // If this is wrong, things will exception neatly later on, so it can't be used
- // to do anything more than cause an error on downloading.
- mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
- mpClearData = (uint8_t*)::malloc(mClearDataSize);
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
-// Purpose: Read the block index from the stream, and store in internal buffer (minus header)
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
-{
- // Header
- file_BlockIndexHeader blkhdr;
-
- // Read it in -- way depends on how whether the magic number has already been read
- if(MagicAlreadyRead)
- {
- // Read the header, without the magic number
- if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
- 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
- }
- else
- {
- // Magic not already read, so need to read the full header
- if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
-
- // Check magic value
- if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
-#endif
- )
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
- }
-
- // Get the number of blocks out of the header
- mNumBlocks = box_ntoh64(blkhdr.mNumBlocks);
-
- // Read the IV base
- mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase);
-
- // Load the block entries in?
- if(mNumBlocks > 0)
- {
- // How big is the index?
- int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
-
- // Allocate some memory
- mpBlockIndex = ::malloc(indexSize);
- if(mpBlockIndex == 0)
- {
- throw std::bad_alloc();
- }
-
- // Read it in
- if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::Read(void *, int, int)
-// Purpose: As interface. Reads decrpyted data.
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
-{
- // Symlinks don't have data. So can't read it. Not even zero bytes.
- if(IsSymLink())
- {
- // Don't allow reading in this case
- THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
- }
-
- // Already finished?
- if(mCurrentBlock >= mNumBlocks)
- {
- // At end of stream, nothing to do
- return 0;
- }
-
- int bytesToRead = NBytes;
- uint8_t *output = (uint8_t*)pBuffer;
-
- while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
- {
- // Anything left in the current block?
- if(mPositionInCurrentBlock < mCurrentBlockClearSize)
- {
- // Copy data out of this buffer
- int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
- if(s > bytesToRead) s = bytesToRead; // limit to requested data
-
- // Copy
- ::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
-
- // Update positions
- output += s;
- mPositionInCurrentBlock += s;
- bytesToRead -= s;
- }
-
- // Need to get some more data?
- if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
- {
- // Number of next block
- ++mCurrentBlock;
- if(mCurrentBlock >= mNumBlocks)
- {
- // Stop now!
- break;
- }
-
- // Get the size from the block index
- const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
- int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize);
- if(encodedSize <= 0)
- {
- // The caller is attempting to decode a file which is the direct result of a diff
- // operation, and so does not contain all the data.
- // It needs to be combined with the previous version first.
- THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
- }
-
- // Load in next block
- if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
- }
-
- // Decode the data
- mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
-
- // Calculate IV for this entry
- uint64_t iv = mEntryIVBase;
- iv += mCurrentBlock;
- // Convert to network byte order before encrypting with it, so that restores work on
- // platforms with different endiannesses.
- iv = box_hton64(iv);
- sBlowfishDecryptBlockEntry.SetIV(&iv);
-
- // Decrypt the encrypted section
- file_BlockIndexEntryEnc entryEnc;
- int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
- entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
- if(sectionSize != sizeof(entryEnc))
- {
- THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
- }
-
- // Make sure this is the right size
- if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
- {
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- if(!mIsOldVersion)
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
- // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
- // IV for the encrypted section. Try again, with the thing the other way round
- iv = box_swap64(iv);
- sBlowfishDecryptBlockEntry.SetIV(&iv);
- int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
- entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
- if(sectionSize != sizeof(entryEnc))
- {
- THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
- }
- if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
- else
- {
- // Warn and log this issue
- if(!sWarnedAboutBackwardsCompatiblity)
- {
- BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index.");
- sWarnedAboutBackwardsCompatiblity = true;
- }
- }
-#else
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
-#endif
- }
-
- // Check the digest
- MD5Digest md5;
- md5.Add(mpClearData, mCurrentBlockClearSize);
- md5.Finish();
- if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
- {
- THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
- }
-
- // Set vars to say what's happening
- mPositionInCurrentBlock = 0;
- }
- }
-
- ASSERT(bytesToRead >= 0);
- ASSERT(bytesToRead <= NBytes);
-
- return NBytes - bytesToRead;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::IsSymLink()
-// Purpose: Is the unencoded file actually a symlink?
-// Created: 10/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFile::DecodedStream::IsSymLink()
-{
- // First, check in with the attributes
- if(!mAttributes.IsSymLink())
- {
- return false;
- }
-
- // So the attributes think it is a symlink.
- // Consistency check...
- if(mNumBlocks != 0)
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- return true;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::Write(const void *, int)
-// Purpose: As interface. Throws exception, as you can't write to this stream.
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
-{
- THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::StreamDataLeft()
-// Purpose: As interface. Any data left?
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFile::DecodedStream::StreamDataLeft()
-{
- return mCurrentBlock < mNumBlocks;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodedStream::StreamClosed()
-// Purpose: As interface. Always returns true, no writing allowed.
-// Created: 9/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFile::DecodedStream::StreamClosed()
-{
- // Can't write to this stream!
- return true;
-}
-
-
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::SetBlowfishKey(const void *, int)
-// Purpose: Static. Sets the key to use for encryption and decryption.
-// Created: 7/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
-{
- // IVs set later
- sBlowfishEncrypt.Reset();
- sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
- sBlowfishDecrypt.Reset();
- sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
-
- sBlowfishEncryptBlockEntry.Reset();
- sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
- sBlowfishEncryptBlockEntry.UsePadding(false);
- sBlowfishDecryptBlockEntry.Reset();
- sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
- sBlowfishDecryptBlockEntry.UsePadding(false);
-}
-
-
-#ifndef HAVE_OLD_SSL
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::SetAESKey(const void *, int)
-// Purpose: Sets the AES key to use for file data encryption. Will select AES as
-// the cipher to use when encrypting.
-// Created: 27/4/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
-{
- // Setup context
- sAESEncrypt.Reset();
- sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
- sAESDecrypt.Reset();
- sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
-
- // Set encryption to use this key, instead of the "default" blowfish key
- spEncrypt = &sAESEncrypt;
- sEncryptCipherType = HEADER_AES_ENCODING;
-}
-#endif
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int)
-// Purpose: The maximum output size of a block, given the chunk size
-// Created: 7/12/03
-//
-// --------------------------------------------------------------------------
-int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
-{
- // Calculate... the maximum size of output by first the largest it could be after compression,
- // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
- // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
- // an problem where a rather non-compressable file didn't fit in a block buffer.)
- return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
- + sBlowfishEncrypt.GetIVLength() + 128;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
-// Purpose: Encodes a chunk (encryption, possible compressed beforehand)
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
-{
- ASSERT(spEncrypt != 0);
-
- // Check there's some space in the output block
- if(rOutput.mBufferSize < 256)
- {
- rOutput.Reallocate(256);
- }
-
- // Check alignment of the block
- ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
-
- // Want to compress it?
- bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
-
- // Build header
- uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
- if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
-
- // Store header
- rOutput.mpBuffer[0] = header;
- int outOffset = 1;
-
- // Setup cipher, and store the IV
- int ivLen = 0;
- const void *iv = spEncrypt->SetRandomIV(ivLen);
- ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
- outOffset += ivLen;
-
- // Start encryption process
- spEncrypt->Begin();
-
- #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \
- { \
- if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \
- { \
- rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \
- } \
- }
-
- // Encode the chunk
- if(compressChunk)
- {
- // buffer to compress into
- uint8_t buffer[2048];
-
- // Set compressor with all the chunk as an input
- Compress<true> compress;
- compress.Input(Chunk, ChunkSize);
- compress.FinishInput();
-
- // Get and encrypt output
- while(!compress.OutputHasFinished())
- {
- int s = compress.Output(buffer, sizeof(buffer));
- if(s > 0)
- {
- ENCODECHUNK_CHECK_SPACE(s)
- outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);
- }
- else
- {
- // Should never happen, as we put all the input in in one go.
- // So if this happens, it means there's a logical problem somewhere
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
- }
- ENCODECHUNK_CHECK_SPACE(16)
- outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
- }
- else
- {
- // Straight encryption
- ENCODECHUNK_CHECK_SPACE(ChunkSize)
- outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
- ENCODECHUNK_CHECK_SPACE(16)
- outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
- }
-
- ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check
-
- return outOffset;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int)
-// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
-// the extra output buffer size needed before calling.
-// See notes in EncodeChunk() for notes re alignment of the
-// encoded data.
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
-{
- // Check alignment of the encoded block
- ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
-
- // First check
- if(EncodedSize < 1)
- {
- THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
- }
-
- const uint8_t *input = (uint8_t*)Encoded;
-
- // Get header, make checks, etc
- uint8_t header = input[0];
- bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
- uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
- if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
- {
- THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
- }
-
-#ifndef HAVE_OLD_SSL
- // Choose cipher
- CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
-#else
- // AES not supported with this version of OpenSSL
- if(encodingType == HEADER_AES_ENCODING)
- {
- THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
- }
- CipherContext &cipher(sBlowfishDecrypt);
-#endif
-
- // Check enough space for header, an IV and one byte of input
- int ivLen = cipher.GetIVLength();
- if(EncodedSize < (1 + ivLen + 1))
- {
- THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
- }
-
- // Set IV in decrypt context, and start
- cipher.SetIV(input + 1);
- cipher.Begin();
-
- // Setup vars for code
- int inOffset = 1 + ivLen;
- uint8_t *output = (uint8_t*)Output;
- int outOffset = 0;
-
- // Do action
- if(chunkCompressed)
- {
- // Do things in chunks
- uint8_t buffer[2048];
- int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
-
- // Decompressor
- Compress<false> decompress;
-
- while(inOffset < EncodedSize)
- {
- // Decrypt a block
- int bl = inputBlockLen;
- if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long
- int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
- inOffset += bl;
-
- // Decompress the decrypted data
- if(s > 0)
- {
- decompress.Input(buffer, s);
- int os = 0;
- do
- {
- os = decompress.Output(output + outOffset, OutputSize - outOffset);
- outOffset += os;
- } while(os > 0);
-
- // Check that there's space left in the output buffer -- there always should be
- if(outOffset >= OutputSize)
- {
- THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
- }
- }
- }
-
- // Get any compressed data remaining in the cipher context and compression
- int s = cipher.Final(buffer, sizeof(buffer));
- decompress.Input(buffer, s);
- decompress.FinishInput();
- while(!decompress.OutputHasFinished())
- {
- int os = decompress.Output(output + outOffset, OutputSize - outOffset);
- outOffset += os;
-
- // Check that there's space left in the output buffer -- there always should be
- if(outOffset >= OutputSize)
- {
- THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
- }
- }
- }
- else
- {
- // Easy decryption
- outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
- outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
- }
-
- return outOffset;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
-// Purpose: Returns a stream which gives a Stream order version of the encoded file.
-// If TakeOwnership == true, then the input stream will be deleted when the
-// returned stream is deleted.
-// The input stream must be seekable.
-// Created: 10/12/03
-//
-// --------------------------------------------------------------------------
-std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
-{
- ASSERT(pStream != 0);
-
- // Get the size of the file
- int64_t fileSize = pStream->BytesLeftToRead();
- if(fileSize == IOStream::SizeOfStreamUnknown)
- {
- THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
- }
-
- // Read the header
- int bytesRead = 0;
- file_StreamFormat hdr;
- bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
-
- // Seek backwards to put the file pointer back where it was before we started this
- pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
-
- // Check we got a block
- if(!readBlock)
- {
- // Couldn't read header -- assume file bad
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // Check magic number
- if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
-#endif
- )
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // Get number of blocks
- int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
-
- // Calculate where the block index will be, check it's reasonable
- int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
- int64_t blockIndexLoc = fileSize - blockIndexSize;
- if(blockIndexLoc < 0)
- {
- // Doesn't look good!
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // Build a reordered stream
- std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
-
- // Set it up...
- ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
- int component = rreordered.AddComponent(pStream);
- // Send out the block index
- rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
- // And then the rest of the file
- rreordered.AddBlock(component, blockIndexLoc, true, 0);
-
- return reordered;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::ResetStats()
-// Purpose: Reset the gathered statistics
-// Created: 20/1/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::ResetStats()
-{
- msStats.mBytesInEncodedFiles = 0;
- msStats.mBytesAlreadyOnServer = 0;
- msStats.mTotalFileStreamSize = 0;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
-// Purpose: Compares the contents of a file against the checksums contained in the
-// block index. Returns true if the checksums match, meaning the file is
-// extremely likely to match the original. Will always consume the entire index.
-// Created: 21/1/04
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
-{
- // is it a symlink?
- bool sourceIsSymlink = false;
- {
- EMU_STRUCT_STAT st;
- if(EMU_LSTAT(Filename, &st) == -1)
- {
- THROW_EXCEPTION(CommonException, OSFileError)
- }
- if((st.st_mode & S_IFMT) == S_IFLNK)
- {
- sourceIsSymlink = true;
- }
- }
-
- // Open file, if it's not a symlink
- std::auto_ptr<FileStream> in;
- if(!sourceIsSymlink)
- {
- in.reset(new FileStream(Filename));
- }
-
- // Read header
- file_BlockIndexHeader hdr;
- if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
- {
- // Couldn't read header
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Check magic
- if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
-#endif
- )
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
-#endif
-
- // Get basic information
- int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
- uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase);
-
- //TODO: Verify that these sizes look reasonable
-
- // setup
- void *data = 0;
- int32_t dataSize = -1;
- bool matches = true;
- int64_t totalSizeInBlockIndex = 0;
-
- try
- {
- for(int64_t b = 0; b < numBlocks; ++b)
- {
- // Read an entry from the stream
- file_BlockIndexEntry entry;
- if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
- {
- // Couldn't read entry
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Calculate IV for this entry
- uint64_t iv = entryIVBase;
- iv += b;
- iv = box_hton64(iv);
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- if(isOldVersion)
- {
- // Reverse the IV for compatibility
- iv = box_swap64(iv);
- }
-#endif
- sBlowfishDecryptBlockEntry.SetIV(&iv);
-
- // Decrypt the encrypted section
- file_BlockIndexEntryEnc entryEnc;
- int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
- entry.mEnEnc, sizeof(entry.mEnEnc));
- if(sectionSize != sizeof(entryEnc))
- {
- THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
- }
-
- // Size of block
- int32_t blockClearSize = ntohl(entryEnc.mSize);
- if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
- totalSizeInBlockIndex += blockClearSize;
-
- // Make sure there's enough memory allocated to load the block in
- if(dataSize < blockClearSize)
- {
- // Too small, free the block if it's already allocated
- if(data != 0)
- {
- ::free(data);
- data = 0;
- }
- // Allocate a block
- data = ::malloc(blockClearSize + 128);
- if(data == 0)
- {
- throw std::bad_alloc();
- }
- dataSize = blockClearSize + 128;
- }
-
- // Load in the block from the file, if it's not a symlink
- if(!sourceIsSymlink)
- {
- if(in->Read(data, blockClearSize) != blockClearSize)
- {
- // Not enough data left in the file, can't possibly match
- matches = false;
- }
- else
- {
- // Check the checksum
- MD5Digest md5;
- md5.Add(data, blockClearSize);
- md5.Finish();
- if(!md5.DigestMatches(entryEnc.mStrongChecksum))
- {
- // Checksum didn't match
- matches = false;
- }
- }
- }
-
- // Keep on going regardless, to make sure the entire block index stream is read
- // -- must always be consistent about what happens with the stream.
- }
- }
- catch(...)
- {
- // clean up in case of errors
- if(data != 0)
- {
- ::free(data);
- data = 0;
- }
- throw;
- }
-
- // free block
- if(data != 0)
- {
- ::free(data);
- data = 0;
- }
-
- // Check for data left over if it's not a symlink
- if(!sourceIsSymlink)
- {
- // Anything left to read in the file?
- if(in->BytesLeftToRead() != 0)
- {
- // File has extra data at the end
- matches = false;
- }
- }
-
- // Symlinks must have zero size on server
- if(sourceIsSymlink)
- {
- matches = (totalSizeInBlockIndex == 0);
- }
-
- return matches;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer()
-// Purpose: Constructor
-// Created: 25/11/04
-//
-// --------------------------------------------------------------------------
-BackupStoreFile::EncodingBuffer::EncodingBuffer()
- : mpBuffer(0),
- mBufferSize(0)
-{
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer()
-// Purpose: Destructor
-// Created: 25/11/04
-//
-// --------------------------------------------------------------------------
-BackupStoreFile::EncodingBuffer::~EncodingBuffer()
-{
- if(mpBuffer != 0)
- {
- BackupStoreFile::CodingChunkFree(mpBuffer);
- mpBuffer = 0;
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::EncodingBuffer::Allocate(int)
-// Purpose: Do initial allocation of block
-// Created: 25/11/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::EncodingBuffer::Allocate(int Size)
-{
- ASSERT(mpBuffer == 0);
- uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
- if(buffer == 0)
- {
- throw std::bad_alloc();
- }
- mpBuffer = buffer;
- mBufferSize = Size;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::EncodingBuffer::Reallocate(int)
-// Purpose: Reallocate the block. Try not to call this, it has to copy
-// the entire contents as the block can't be reallocated straight.
-// Created: 25/11/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
-{
- BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize <<
- " to " << NewSize);
- ASSERT(mpBuffer != 0);
- uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
- if(buffer == 0)
- {
- throw std::bad_alloc();
- }
- // Copy data
- ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
-
- // Free old
- BackupStoreFile::CodingChunkFree(mpBuffer);
-
- // Store new buffer
- mpBuffer = buffer;
- mBufferSize = NewSize;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: DiffTimer::DiffTimer();
-// Purpose: Constructor
-// Created: 2005/02/01
-//
-// --------------------------------------------------------------------------
-DiffTimer::DiffTimer()
-{
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: DiffTimer::DiffTimer();
-// Purpose: Destructor
-// Created: 2005/02/01
-//
-// --------------------------------------------------------------------------
-DiffTimer::~DiffTimer()
-{
-}
Deleted: box/trunk/lib/backupclient/BackupStoreFile.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFile.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFile.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,231 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFile.h
-// Purpose: Utils for manipulating files
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREFILE__H
-#define BACKUPSTOREFILE__H
-
-#include <cstdlib>
-#include <memory>
-#include <cstdlib>
-
-#include "BackupClientFileAttributes.h"
-#include "BackupStoreFilename.h"
-#include "IOStream.h"
-#include "ReadLoggingStream.h"
-#include "RunStatusProvider.h"
-
-typedef struct
-{
- int64_t mBytesInEncodedFiles;
- int64_t mBytesAlreadyOnServer;
- int64_t mTotalFileStreamSize;
-} BackupStoreFileStats;
-
-// Uncomment to disable backwards compatibility
-//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
-
-
-// Output buffer to EncodeChunk and input data to DecodeChunk must
-// have specific alignment, see function comments.
-#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16
-#define BACKUPSTOREFILE_CODING_OFFSET 15
-
-// Have some memory allocation commands, note closing "Off" at end of file.
-#include "MemLeakFindOn.h"
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: DiffTimer
-// Purpose: Interface for classes that can keep track of diffing time,
-// and send SSL keepalive messages
-// Created: 2006/01/19
-//
-// --------------------------------------------------------------------------
-class DiffTimer
-{
-public:
- DiffTimer();
- virtual ~DiffTimer();
-public:
- virtual void DoKeepAlive() = 0;
- virtual int GetMaximumDiffingTime() = 0;
- virtual bool IsManaged() = 0;
-};
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupStoreFile
-// Purpose: Class to hold together utils for manipulating files.
-// Created: 2003/08/28
-//
-// --------------------------------------------------------------------------
-class BackupStoreFile
-{
-public:
- class DecodedStream : public IOStream
- {
- friend class BackupStoreFile;
- private:
- DecodedStream(IOStream &rEncodedFile, int Timeout);
- DecodedStream(const DecodedStream &); // not allowed
- DecodedStream &operator=(const DecodedStream &); // not allowed
- public:
- ~DecodedStream();
-
- // Stream functions
- virtual int Read(void *pBuffer, int NBytes, int Timeout);
- virtual void Write(const void *pBuffer, int NBytes);
- virtual bool StreamDataLeft();
- virtual bool StreamClosed();
-
- // Accessor functions
- const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
- const BackupStoreFilename &GetFilename() {return mFilename;}
- int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests
-
- bool IsSymLink();
-
- private:
- void Setup(const BackupClientFileAttributes *pAlterativeAttr);
- void ReadBlockIndex(bool MagicAlreadyRead);
-
- private:
- IOStream &mrEncodedFile;
- int mTimeout;
- BackupClientFileAttributes mAttributes;
- BackupStoreFilename mFilename;
- int64_t mNumBlocks;
- void *mpBlockIndex;
- uint8_t *mpEncodedData;
- uint8_t *mpClearData;
- int mClearDataSize;
- int mCurrentBlock;
- int mCurrentBlockClearSize;
- int mPositionInCurrentBlock;
- uint64_t mEntryIVBase;
-#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
- bool mIsOldVersion;
-#endif
- };
-
-
- // Main interface
- static std::auto_ptr<IOStream> EncodeFile
- (
- const char *Filename,
- int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
- int64_t *pModificationTime = 0,
- ReadLoggingStream::Logger* pLogger = NULL,
- RunStatusProvider* pRunStatusProvider = NULL
- );
- static std::auto_ptr<IOStream> EncodeFileDiff
- (
- const char *Filename, int64_t ContainerID,
- const BackupStoreFilename &rStoreFilename,
- int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
- int Timeout,
- DiffTimer *pDiffTimer,
- int64_t *pModificationTime = 0,
- bool *pIsCompletelyDifferent = 0
- );
- static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
- static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
- static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
- static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
- static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
- static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
- static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
- static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
-
- // Stream manipulation
- static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
- static void MoveStreamPositionToBlockIndex(IOStream &rStream);
-
- // Crypto setup
- static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
-#ifndef HAVE_OLD_SSL
- static void SetAESKey(const void *pKey, int KeyLength);
-#endif
-
- // Allocation of properly aligning chunks for decoding and encoding chunks
- inline static void *CodingChunkAlloc(int Size)
- {
- uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
- if(a == 0) return 0;
- // Align to main block size
- ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
- uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE
- - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
- uint8_t *b = (a + adjustment);
- // Store adjustment
- *b = adjustment;
- // Return offset
- return b + BACKUPSTOREFILE_CODING_OFFSET;
- }
- inline static void CodingChunkFree(void *Block)
- {
- // Check alignment is as expected
- ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
- ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
- uint8_t *a = (uint8_t*)Block;
- a -= BACKUPSTOREFILE_CODING_OFFSET;
- // Adjust downwards...
- a -= *a;
- free(a);
- }
-
- static void DiffTimerExpired();
-
- // Building blocks
- class EncodingBuffer
- {
- public:
- EncodingBuffer();
- ~EncodingBuffer();
- private:
- // No copying
- EncodingBuffer(const EncodingBuffer &);
- EncodingBuffer &operator=(const EncodingBuffer &);
- public:
- void Allocate(int Size);
- void Reallocate(int NewSize);
-
- uint8_t *mpBuffer;
- int mBufferSize;
- };
- static int MaxBlockSizeForChunkSize(int ChunkSize);
- static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
-
- // Caller should know how big the output size is, but also allocate a bit more memory to cover various
- // overheads allowed for in checks
- static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
- {
- // Plenty big enough
- return KnownChunkSize + 256;
- }
- static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
-
- // Statisitics, not designed to be completely reliable
- static void ResetStats();
- static BackupStoreFileStats msStats;
-
- // For debug
-#ifndef BOX_RELEASE_BUILD
- static bool TraceDetailsOfDiffProcess;
-#endif
-
- // For decoding encoded files
- static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
-};
-
-#include "MemLeakFindOff.h"
-
-#endif // BACKUPSTOREFILE__H
Deleted: box/trunk/lib/backupclient/BackupStoreFileCryptVar.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileCryptVar.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFileCryptVar.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,31 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFileCryptVar.cpp
-// Purpose: Cryptographic keys for backup store files
-// Created: 12/1/04
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include "BackupStoreFileCryptVar.h"
-#include "BackupStoreFileWire.h"
-
-#include "MemLeakFindOn.h"
-
-CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt;
-CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt;
-
-#ifndef HAVE_OLD_SSL
- CipherContext BackupStoreFileCryptVar::sAESEncrypt;
- CipherContext BackupStoreFileCryptVar::sAESDecrypt;
-#endif
-
-// Default to blowfish
-CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt;
-uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING;
-
-CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry;
-CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry;
-
Deleted: box/trunk/lib/backupclient/BackupStoreFileCryptVar.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileCryptVar.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFileCryptVar.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,39 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFileCryptVar.h
-// Purpose: Cryptographic keys for backup store files
-// Created: 12/1/04
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREFILECRYPTVAR__H
-#define BACKUPSTOREFILECRYPTVAR__H
-
-#include "CipherContext.h"
-
-// Hide private static variables from the rest of the world by putting them
-// as static variables in a namespace.
-// -- don't put them as static class variables to avoid openssl/evp.h being
-// included all over the project.
-namespace BackupStoreFileCryptVar
-{
- // Keys for the main file data
- extern CipherContext sBlowfishEncrypt;
- extern CipherContext sBlowfishDecrypt;
- // Use AES when available
-#ifndef HAVE_OLD_SSL
- extern CipherContext sAESEncrypt;
- extern CipherContext sAESDecrypt;
-#endif
- // How encoding will be done
- extern CipherContext *spEncrypt;
- extern uint8_t sEncryptCipherType;
-
- // Keys for the block indicies
- extern CipherContext sBlowfishEncryptBlockEntry;
- extern CipherContext sBlowfishDecryptBlockEntry;
-}
-
-#endif // BACKUPSTOREFILECRYPTVAR__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreFileEncodeStream.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileEncodeStream.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFileEncodeStream.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,717 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFileEncodeStream.cpp
-// Purpose: Implement stream-based file encoding for the backup store
-// Created: 12/1/04
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include <string.h>
-
-#include "BackupClientFileAttributes.h"
-#include "BackupStoreConstants.h"
-#include "BackupStoreException.h"
-#include "BackupStoreFile.h"
-#include "BackupStoreFileCryptVar.h"
-#include "BackupStoreFileEncodeStream.h"
-#include "BackupStoreFileWire.h"
-#include "BackupStoreObjectMagic.h"
-#include "BoxTime.h"
-#include "FileStream.h"
-#include "Random.h"
-#include "RollingChecksum.h"
-
-#include "MemLeakFindOn.h"
-
-#include <cstring>
-
-using namespace BackupStoreFileCryptVar;
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
-// Purpose: Constructor (opens file)
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
- : mpRecipe(0),
- mpFile(0),
- mpLogging(0),
- mpRunStatusProvider(NULL),
- mStatus(Status_Header),
- mSendData(true),
- mTotalBlocks(0),
- mAbsoluteBlockNumber(-1),
- mInstructionNumber(-1),
- mNumBlocks(0),
- mCurrentBlock(-1),
- mCurrentBlockEncodedSize(0),
- mPositionInCurrentBlock(0),
- mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
- mLastBlockSize(0),
- mTotalBytesSent(0),
- mpRawBuffer(0),
- mAllocatedBufferSize(0),
- mEntryIVBase(0)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
-// Purpose: Destructor
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
-{
- // Free buffers
- if(mpRawBuffer)
- {
- ::free(mpRawBuffer);
- mpRawBuffer = 0;
- }
-
- // Close the file, which we might have open
- if(mpFile)
- {
- delete mpFile;
- mpFile = 0;
- }
-
- // Clear up logging stream
- if(mpLogging)
- {
- delete mpLogging;
- mpLogging = 0;
- }
-
- // Free the recipe
- if(mpRecipe != 0)
- {
- delete mpRecipe;
- mpRecipe = 0;
- }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
-// Purpose: Reads file information, and builds file header reading for sending.
-// Takes ownership of the Recipe.
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::Setup(const char *Filename,
- BackupStoreFileEncodeStream::Recipe *pRecipe,
- int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
- int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger,
- RunStatusProvider* pRunStatusProvider)
-{
- // Pointer to a blank recipe which we might create
- BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
-
- try
- {
- // Get file attributes
- box_time_t modTime = 0;
- int64_t fileSize = 0;
- BackupClientFileAttributes attr;
- attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
- 0 /* not interested in attr mod time */, &fileSize);
-
- // Might need to create a blank recipe...
- if(pRecipe == 0)
- {
- pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
-
- BackupStoreFileEncodeStream::RecipeInstruction instruction;
- instruction.mSpaceBefore = fileSize; // whole file
- instruction.mBlocks = 0; // no blocks
- instruction.mpStartBlock = 0; // no block
- pblankRecipe->push_back(instruction);
-
- pRecipe = pblankRecipe;
- }
-
- // Tell caller?
- if(pModificationTime != 0)
- {
- *pModificationTime = modTime;
- }
-
- // Go through each instruction in the recipe and work out how many blocks
- // it will add, and the max clear size of these blocks
- int maxBlockClearSize = 0;
- for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
- {
- if((*pRecipe)[inst].mSpaceBefore > 0)
- {
- // Calculate the number of blocks the space before requires
- int64_t numBlocks;
- int32_t blockSize, lastBlockSize;
- CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
- // Add to accumlated total
- mTotalBlocks += numBlocks;
- // Update maximum clear size
- if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
- if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
- }
-
- // Add number of blocks copied from the previous file
- mTotalBlocks += (*pRecipe)[inst].mBlocks;
-
- // Check for bad things
- if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
- {
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-
- // Run through blocks to get the max clear size
- for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
- {
- if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
- }
- }
-
- // Send data? (symlinks don't have any data in them)
- mSendData = !attr.IsSymLink();
-
- // If not data is being sent, then the max clear block size is zero
- if(!mSendData)
- {
- maxBlockClearSize = 0;
- }
-
- // Header
- file_StreamFormat hdr;
- hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
- hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0);
- hdr.mContainerID = box_hton64(ContainerID);
- hdr.mModificationTime = box_hton64(modTime);
- // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
- hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
- hdr.mOptions = 0; // no options defined yet
-
- // Write header to stream
- mData.Write(&hdr, sizeof(hdr));
-
- // Write filename to stream
- rStoreFilename.WriteToStream(mData);
-
- // Write attributes to stream
- attr.WriteToStream(mData);
-
- // Allocate some buffers for writing data
- if(mSendData)
- {
- // Open the file
- mpFile = new FileStream(Filename);
-
- if (pLogger)
- {
- // Create logging stream
- mpLogging = new ReadLoggingStream(*mpFile,
- *pLogger);
- }
- else
- {
- // re-use FileStream instead
- mpLogging = mpFile;
- mpFile = NULL;
- }
-
- // Work out the largest possible block required for the encoded data
- mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
-
- // Then allocate two blocks of this size
- mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
- if(mpRawBuffer == 0)
- {
- throw std::bad_alloc();
- }
-#ifndef BOX_RELEASE_BUILD
- // In debug builds, make sure that the reallocation code is exercised.
- mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
-#else
- mEncodedBuffer.Allocate(mAllocatedBufferSize);
-#endif
- }
- else
- {
- // Write an empty block index for the symlink
- file_BlockIndexHeader blkhdr;
- blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
- blkhdr.mOtherFileID = box_hton64(0); // not other file ID
- blkhdr.mEntryIVBase = box_hton64(0);
- blkhdr.mNumBlocks = box_hton64(0);
- mData.Write(&blkhdr, sizeof(blkhdr));
- }
-
- // Ready for reading
- mData.SetForReading();
-
- // Update stats
- BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
-
- // Finally, store the pointer to the recipe, when we know exceptions won't occur
- mpRecipe = pRecipe;
- }
- catch(...)
- {
- // Clean up any blank recipe
- if(pblankRecipe != 0)
- {
- delete pblankRecipe;
- pblankRecipe = 0;
- }
- throw;
- }
-
- mpRunStatusProvider = pRunStatusProvider;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
-// Purpose: Calculates the sizes of blocks in a section of the file
-// Created: 16/1/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
-{
- // How many blocks, and how big?
- rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
- do
- {
- rBlockSizeOut *= 2;
-
- rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
-
- } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
-
- // Last block size
- rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
-
- // Avoid small blocks?
- if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
- && rNumBlocksOut > 1)
- {
- // Add the small bit of data to the last block
- --rNumBlocksOut;
- rLastBlockSizeOut += rBlockSizeOut;
- }
-
- // checks!
- ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
- //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::Read(void *, int, int)
-// Purpose: As interface -- generates encoded file data on the fly from the raw file
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
-{
- // Check there's something to do.
- if(mStatus == Status_Finished)
- {
- return 0;
- }
-
- if(mpRunStatusProvider && mpRunStatusProvider->StopRun())
- {
- THROW_EXCEPTION(BackupStoreException, SignalReceived);
- }
-
- int bytesToRead = NBytes;
- uint8_t *buffer = (uint8_t*)pBuffer;
-
- while(bytesToRead > 0 && mStatus != Status_Finished)
- {
- if(mStatus == Status_Header || mStatus == Status_BlockListing)
- {
- // Header or block listing phase -- send from the buffered stream
-
- // Send bytes from the data buffer
- int b = mData.Read(buffer, bytesToRead, Timeout);
- bytesToRead -= b;
- buffer += b;
-
- // Check to see if all the data has been used from this stream
- if(!mData.StreamDataLeft())
- {
- // Yes, move on to next phase (or finish, if there's no file data)
- if(!mSendData)
- {
- mStatus = Status_Finished;
- }
- else
- {
- // Reset the buffer so it can be used for the next phase
- mData.Reset();
-
- // Get buffer ready for index?
- if(mStatus == Status_Header)
- {
- // Just finished doing the stream header, create the block index header
- file_BlockIndexHeader blkhdr;
- blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
- ASSERT(mpRecipe != 0);
- blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID());
- blkhdr.mNumBlocks = box_hton64(mTotalBlocks);
-
- // Generate the IV base
- Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
- blkhdr.mEntryIVBase = box_hton64(mEntryIVBase);
-
- mData.Write(&blkhdr, sizeof(blkhdr));
- }
-
- ++mStatus;
- }
- }
- }
- else if(mStatus == Status_Blocks)
- {
- // Block sending phase
-
- if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
- {
- // Next block!
- ++mCurrentBlock;
- ++mAbsoluteBlockNumber;
- if(mCurrentBlock >= mNumBlocks)
- {
- // Output extra blocks for this instruction and move forward in file
- if(mInstructionNumber >= 0)
- {
- SkipPreviousBlocksInInstruction();
- }
-
- // Is there another instruction to go?
- ++mInstructionNumber;
-
- // Skip instructions which don't contain any data
- while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
- && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
- {
- SkipPreviousBlocksInInstruction();
- ++mInstructionNumber;
- }
-
- if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
- {
- // End of blocks, go to next phase
- ++mStatus;
-
- // Set the data to reading so the index can be written
- mData.SetForReading();
- }
- else
- {
- // Get ready for this instruction
- SetForInstruction();
- }
- }
-
- // Can't use 'else' here as SetForInstruction() will change this
- if(mCurrentBlock < mNumBlocks)
- {
- EncodeCurrentBlock();
- }
- }
-
- // Send data from the current block (if there's data to send)
- if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
- {
- // How much data to put in the buffer?
- int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
- if(s > bytesToRead) s = bytesToRead;
-
- // Copy it in
- ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
-
- // Update variables
- bytesToRead -= s;
- buffer += s;
- mPositionInCurrentBlock += s;
- }
- }
- else
- {
- // Should never get here, as it'd be an invalid status
- ASSERT(false);
- }
- }
-
- // Add encoded size to stats
- BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
- mTotalBytesSent += (NBytes - bytesToRead);
-
- // Return size of data to caller
- return NBytes - bytesToRead;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
-// Purpose: Private. Stores the blocks of the old file referenced in the current
-// instruction into the index and skips over the data in the file
-// Created: 16/1/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
-{
- // Check something is necessary
- if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
- {
- return;
- }
-
- // Index of the first block in old file (being diffed from)
- int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
-
- int64_t sizeToSkip = 0;
-
- for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
- {
- // Update stats
- BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
-
- // Store the entry
- StoreBlockIndexEntry(0 - (firstIndex + b),
- (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
- (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
- (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);
-
- // Increment the absolute block number -- kept encryption IV in sync
- ++mAbsoluteBlockNumber;
-
- // Add the size of this block to the size to skip
- sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
- }
-
- // Move forward in the stream
- mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::SetForInstruction()
-// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
-// Created: 16/1/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::SetForInstruction()
-{
- // Calculate block sizes
- CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
-
- // Set variables
- mCurrentBlock = 0;
- mCurrentBlockEncodedSize = 0;
- mPositionInCurrentBlock = 0;
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock()
-// Purpose: Private. Encodes the current block, and writes the block data to the index
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::EncodeCurrentBlock()
-{
- // How big is the block, raw?
- int blockRawSize = mBlockSize;
- if(mCurrentBlock == (mNumBlocks - 1))
- {
- blockRawSize = mLastBlockSize;
- }
- ASSERT(blockRawSize < mAllocatedBufferSize);
-
- // Check file open
- if(mpLogging == 0)
- {
- // File should be open, but isn't. So logical error.
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-
- // Read the data in
- if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize,
- 0 /* not interested in size if failure */))
- {
- // TODO: Do something more intelligent, and abort
- // this upload because the file has changed.
- THROW_EXCEPTION(BackupStoreException,
- Temp_FileEncodeStreamDidntReadBuffer)
- }
-
- // Encode it
- mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer,
- blockRawSize, mEncodedBuffer);
-
- //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
-
- // Create block listing data -- generate checksums
- RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
- MD5Digest strongChecksum;
- strongChecksum.Add(mpRawBuffer, blockRawSize);
- strongChecksum.Finish();
-
- // Add entry to the index
- StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize,
- weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
-
- // Set vars to reading this block
- mPositionInCurrentBlock = 0;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
-// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
-// Created: 16/1/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
-{
- // First, the encrypted section
- file_BlockIndexEntryEnc entryEnc;
- entryEnc.mSize = htonl(ClearSize);
- entryEnc.mWeakChecksum = htonl(WeakChecksum);
- ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
-
- // Then the clear section
- file_BlockIndexEntry entry;
- entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex));
-
- // Then encrypt the encryted section
- // Generate the IV from the block number
- if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
- {
- THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
- }
- uint64_t iv = mEntryIVBase;
- iv += mAbsoluteBlockNumber;
- // Convert to network byte order before encrypting with it, so that restores work on
- // platforms with different endiannesses.
- iv = box_hton64(iv);
- sBlowfishEncryptBlockEntry.SetIV(&iv);
-
- // Encode the data
- int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
- if(encodedSize != sizeof(entry.mEnEnc))
- {
- THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
- }
-
- // Save to data block for sending at the end of the stream
- mData.Write(&entry, sizeof(entry));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::Write(const void *, int)
-// Purpose: As interface. Exceptions.
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
-{
- THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::StreamDataLeft()
-// Purpose: As interface -- end of stream reached?
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFileEncodeStream::StreamDataLeft()
-{
- return (mStatus != Status_Finished);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::StreamClosed()
-// Purpose: As interface
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFileEncodeStream::StreamClosed()
-{
- return true;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
-// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
-// Created: 15/1/04
-//
-// --------------------------------------------------------------------------
-BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
- int64_t NumBlocksInIndex, int64_t OtherFileID)
- : mpBlockIndex(pBlockIndex),
- mNumBlocksInIndex(NumBlocksInIndex),
- mOtherFileID(OtherFileID)
-{
- ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFileEncodeStream::Recipe::~Recipe()
-// Purpose: Destructor
-// Created: 15/1/04
-//
-// --------------------------------------------------------------------------
-BackupStoreFileEncodeStream::Recipe::~Recipe()
-{
- // Free the block index, if there is one
- if(mpBlockIndex != 0)
- {
- ::free(mpBlockIndex);
- }
-}
-
-
-
-
Deleted: box/trunk/lib/backupclient/BackupStoreFileEncodeStream.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileEncodeStream.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFileEncodeStream.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,137 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFileEncodeStream.h
-// Purpose: Implement stream-based file encoding for the backup store
-// Created: 12/1/04
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREFILEENCODESTREAM__H
-#define BACKUPSTOREFILEENCODESTREAM__H
-
-#include <vector>
-
-#include "IOStream.h"
-#include "BackupStoreFilename.h"
-#include "CollectInBufferStream.h"
-#include "MD5Digest.h"
-#include "BackupStoreFile.h"
-#include "ReadLoggingStream.h"
-#include "RunStatusProvider.h"
-
-namespace BackupStoreFileCreation
-{
- // Diffing and creation of files share some implementation details.
- typedef struct _BlocksAvailableEntry
- {
- struct _BlocksAvailableEntry *mpNextInHashList;
- int32_t mSize; // size in clear
- uint32_t mWeakChecksum; // weak, rolling checksum
- uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
- } BlocksAvailableEntry;
-
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupStoreFileEncodeStream
-// Purpose: Encode a file into a stream
-// Created: 8/12/03
-//
-// --------------------------------------------------------------------------
-class BackupStoreFileEncodeStream : public IOStream
-{
-public:
- BackupStoreFileEncodeStream();
- ~BackupStoreFileEncodeStream();
-
- typedef struct
- {
- int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go
- int32_t mBlocks; // number of block to reuse, starting at this one
- BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null
- } RecipeInstruction;
-
- class Recipe : public std::vector<RecipeInstruction>
- {
- // NOTE: This class is rather tied in with the implementation of diffing.
- public:
- Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex,
- int64_t OtherFileID = 0);
- ~Recipe();
-
- int64_t GetOtherFileID() {return mOtherFileID;}
- int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock)
- {
- return pBlock - mpBlockIndex;
- }
-
- private:
- BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex;
- int64_t mNumBlocksInIndex;
- int64_t mOtherFileID;
- };
-
- void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID,
- const BackupStoreFilename &rStoreFilename,
- int64_t *pModificationTime,
- ReadLoggingStream::Logger* pLogger = NULL,
- RunStatusProvider* pRunStatusProvider = NULL);
-
- virtual int Read(void *pBuffer, int NBytes, int Timeout);
- virtual void Write(const void *pBuffer, int NBytes);
- virtual bool StreamDataLeft();
- virtual bool StreamClosed();
- int64_t GetTotalBytesSent() { return mTotalBytesSent; }
-
-private:
- enum
- {
- Status_Header = 0,
- Status_Blocks = 1,
- Status_BlockListing = 2,
- Status_Finished = 3
- };
-
-private:
- void EncodeCurrentBlock();
- void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut);
- void SkipPreviousBlocksInInstruction();
- void SetForInstruction();
- void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum);
-
-private:
- Recipe *mpRecipe;
- IOStream *mpFile; // source file
- CollectInBufferStream mData; // buffer for header and index entries
- IOStream *mpLogging;
- RunStatusProvider* mpRunStatusProvider;
- int mStatus;
- bool mSendData; // true if there's file data to send (ie not a symlink)
- int64_t mTotalBlocks; // Total number of blocks in the file
- int64_t mAbsoluteBlockNumber; // The absolute block number currently being output
- // Instruction number
- int64_t mInstructionNumber;
- // All the below are within the current instruction
- int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
- int64_t mCurrentBlock;
- int32_t mCurrentBlockEncodedSize;
- int32_t mPositionInCurrentBlock; // for reading out
- int32_t mBlockSize; // Basic block size of most of the blocks in the file
- int32_t mLastBlockSize; // the size (unencoded) of the last block in the file
- int64_t mTotalBytesSent;
- // Buffers
- uint8_t *mpRawBuffer; // buffer for raw data
- BackupStoreFile::EncodingBuffer mEncodedBuffer;
- // buffer for encoded data
- int32_t mAllocatedBufferSize; // size of above two allocated blocks
- uint64_t mEntryIVBase; // base for block entry IV
-};
-
-
-
-#endif // BACKUPSTOREFILEENCODESTREAM__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreFileRevDiff.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileRevDiff.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFileRevDiff.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,258 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFileRevDiff.cpp
-// Purpose: Reverse a patch, to build a new patch from new to old files
-// Created: 12/7/04
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-
-#include <new>
-#include <stdlib.h>
-
-#include "BackupStoreFile.h"
-#include "BackupStoreFileWire.h"
-#include "BackupStoreObjectMagic.h"
-#include "BackupStoreException.h"
-#include "BackupStoreConstants.h"
-#include "BackupStoreFilename.h"
-
-#include "MemLeakFindOn.h"
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t)
-// Purpose: Reverse a patch, to build a new patch from new to old files. Takes
-// two independent copies to the From file, for efficiency.
-// Created: 12/7/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent)
-{
- // Read and copy the header from the from file to the out file -- beginnings of the patch
- file_StreamFormat hdr;
- if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0))
- {
- THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
- }
- if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
- // Copy
- rOut.Write(&hdr, sizeof(hdr));
- // Copy over filename and attributes
- // BLOCK
- {
- BackupStoreFilename filename;
- filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
- filename.WriteToStream(rOut);
- StreamableMemBlock attr;
- attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
- attr.WriteToStream(rOut);
- }
-
- // Build an index of common blocks.
- // For each block in the from file, we want to know it's index in the
- // diff file. Allocate memory for this information.
- int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks);
- int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t));
- if(pfromIndexInfo == 0)
- {
- throw std::bad_alloc();
- }
-
- // Buffer data
- void *buffer = 0;
- int bufferSize = 0;
-
- // flag
- bool isCompletelyDifferent = true;
-
- try
- {
- // Initialise the index to be all 0, ie not filled in yet
- for(int64_t i = 0; i < fromNumBlocks; ++i)
- {
- pfromIndexInfo[i] = 0;
- }
-
- // Within the from file, skip to the index
- MoveStreamPositionToBlockIndex(rDiff);
-
- // Read in header of index
- file_BlockIndexHeader diffIdxHdr;
- if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
- if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // And then read in each entry
- int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks);
- for(int64_t b = 0; b < diffNumBlocks; ++b)
- {
- file_BlockIndexEntry e;
- if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Where's the block?
- int64_t blockEn = box_ntoh64(e.mEncodedSize);
- if(blockEn > 0)
- {
- // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file
- }
- else
- {
- // Block is in the original file, store which block it is in this file
- int64_t fromIndex = 0 - blockEn;
- if(fromIndex < 0 || fromIndex >= fromNumBlocks)
- {
- THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles)
- }
-
- // Store information about where it is in the new file
- // NOTE: This is slight different to how it'll be stored in the final index.
- pfromIndexInfo[fromIndex] = -1 - b;
- }
- }
-
- // Open the index for the second copy of the from file
- MoveStreamPositionToBlockIndex(rFrom2);
-
- // Read in header of index
- file_BlockIndexHeader fromIdxHdr;
- if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
- if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
- || box_ntoh64(fromIdxHdr.mOtherFileID) != 0)
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // So, we can now start building the data in the file
- int64_t filePosition = rFrom.GetPosition();
- for(int64_t b = 0; b < fromNumBlocks; ++b)
- {
- // Read entry from from index
- file_BlockIndexEntry e;
- if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Get size
- int64_t blockSize = box_hton64(e.mEncodedSize);
- if(blockSize < 0)
- {
- THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
- }
-
- // Copy this block?
- if(pfromIndexInfo[b] == 0)
- {
- // Copy it, first move to file location
- rFrom.Seek(filePosition, IOStream::SeekType_Absolute);
-
- // Make sure there's memory available to copy this
- if(bufferSize < blockSize || buffer == 0)
- {
- // Free old block
- if(buffer != 0)
- {
- ::free(buffer);
- buffer = 0;
- bufferSize = 0;
- }
- // Allocate new block
- buffer = ::malloc(blockSize);
- if(buffer == 0)
- {
- throw std::bad_alloc();
- }
- bufferSize = blockSize;
- }
- ASSERT(bufferSize >= blockSize);
-
- // Copy the block
- if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
- {
- THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
- }
- rOut.Write(buffer, blockSize);
-
- // Store the size
- pfromIndexInfo[b] = blockSize;
- }
- else
- {
- // Block isn't needed, so it's not completely different
- isCompletelyDifferent = false;
- }
- filePosition += blockSize;
- }
-
- // Then write the index, modified header first
- fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom));
- rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr));
-
- // Move to start of index entries
- rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute);
-
- // Then copy modified entries
- for(int64_t b = 0; b < fromNumBlocks; ++b)
- {
- // Read entry from from index
- file_BlockIndexEntry e;
- if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // Modify...
- int64_t s = pfromIndexInfo[b];
- // Adjust to reflect real block index (remember 0 has a different meaning here)
- if(s < 0) ++s;
- // Insert
- e.mEncodedSize = box_hton64(s);
- // Write
- rOut.Write(&e, sizeof(e));
- }
- }
- catch(...)
- {
- ::free(pfromIndexInfo);
- if(buffer != 0)
- {
- ::free(buffer);
- }
- throw;
- }
-
- // Free memory used (oh for finally {} blocks)
- ::free(pfromIndexInfo);
- if(buffer != 0)
- {
- ::free(buffer);
- }
-
- // return completely different flag
- if(pIsCompletelyDifferent != 0)
- {
- *pIsCompletelyDifferent = isCompletelyDifferent;
- }
-}
-
-
-
Deleted: box/trunk/lib/backupclient/BackupStoreFileWire.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFileWire.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFileWire.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,74 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFileWire.h
-// Purpose: On the wire / disc formats for backup store files
-// Created: 12/1/04
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREFILEWIRE__H
-#define BACKUPSTOREFILEWIRE__H
-
-#include "MD5Digest.h"
-
-// set packing to one byte
-#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
-#include "BeginStructPackForWire.h"
-#else
-BEGIN_STRUCTURE_PACKING_FOR_WIRE
-#endif
-
-typedef struct
-{
- int32_t mMagicValue; // also the version number
- int64_t mNumBlocks; // number of blocks contained in the file
- int64_t mContainerID;
- int64_t mModificationTime;
- int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block
- int32_t mOptions; // bitmask of options used
- // Then a BackupStoreFilename
- // Then a BackupClientFileAttributes
-} file_StreamFormat;
-
-typedef struct
-{
- int32_t mMagicValue; // different magic value
- int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index
- uint64_t mEntryIVBase; // base value for block IV
- int64_t mNumBlocks; // repeat of value in file header
-} file_BlockIndexHeader;
-
-typedef struct
-{
- int32_t mSize; // size in clear
- uint32_t mWeakChecksum; // weak, rolling checksum
- uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
-} file_BlockIndexEntryEnc;
-
-typedef struct
-{
- union
- {
- int64_t mEncodedSize; // size encoded, if > 0
- int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0
- };
- uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section
-} file_BlockIndexEntry;
-
-// Use default packing
-#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
-#include "EndStructPackForWire.h"
-#else
-END_STRUCTURE_PACKING_FOR_WIRE
-#endif
-
-// header for blocks of compressed data in files
-#define HEADER_CHUNK_IS_COMPRESSED 1 // bit
-#define HEADER_ENCODING_SHIFT 1 // shift value
-#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7
-#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7
-
-
-#endif // BACKUPSTOREFILEWIRE__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreFilename.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFilename.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFilename.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,281 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFilename.cpp
-// Purpose: Filename for the backup store
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-#include "BackupStoreFilename.h"
-#include "Protocol.h"
-#include "BackupStoreException.h"
-#include "IOStream.h"
-#include "Guards.h"
-
-#include "MemLeakFindOn.h"
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::BackupStoreFilename()
-// Purpose: Default constructor -- creates an invalid filename
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilename::BackupStoreFilename()
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &)
-// Purpose: Copy constructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy)
- : mEncryptedName(rToCopy.mEncryptedName)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::~BackupStoreFilename()
-// Purpose: Destructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilename::~BackupStoreFilename()
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::CheckValid(bool)
-// Purpose: Checks the encoded filename for validity
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const
-{
- bool ok = true;
-
- if(mEncryptedName.size() < 2)
- {
- // Isn't long enough to have a header
- ok = false;
- }
- else
- {
- // Check size is consistent
- unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName);
- if(dsize != mEncryptedName.size())
- {
- ok = false;
- }
-
- // And encoding is an accepted value
- unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName);
- if(encoding < Encoding_Min || encoding > Encoding_Max)
- {
- ok = false;
- }
- }
-
- // Exception?
- if(!ok && ExceptionIfInvalid)
- {
- THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename)
- }
-
- return ok;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::ReadFromProtocol(Protocol &)
-// Purpose: Reads the filename from the protocol object
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol)
-{
- // Read the header
- char hdr[2];
- rProtocol.Read(hdr, 2);
-
- // How big is it?
- int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
-
- // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
- std::string data;
- rProtocol.Read(data, dsize - 2);
-
- // assign to this string, storing the header and the extra data
- mEncryptedName.assign(hdr, 2);
- mEncryptedName.append(data.c_str(), data.size());
-
- // Check it
- CheckValid();
-
- // Alert derived classes
- EncodedFilenameChanged();
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::WriteToProtocol(Protocol &)
-// Purpose: Writes the filename to the protocol object
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const
-{
- CheckValid();
-
- rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size());
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::ReadFromStream(IOStream &)
-// Purpose: Reads the filename from a stream
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout)
-{
- // Read the header
- char hdr[2];
- if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
-
- // How big is it?
- unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
-
- // Assume most filenames are small
- char buf[256];
- if(dsize < sizeof(buf))
- {
- // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
- if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
- // Copy in header
- buf[0] = hdr[0]; buf[1] = hdr[1];
-
- // assign to this string, storing the header and the extra data
- mEncryptedName.assign(buf, dsize);
- }
- else
- {
- // Block of memory to hold it
- MemoryBlockGuard<char*> dataB(dsize+2);
- char *data = dataB;
-
- // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
- if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
- {
- THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
- }
- // Copy in header
- data[0] = hdr[0]; data[1] = hdr[1];
-
- // assign to this string, storing the header and the extra data
- mEncryptedName.assign(data, dsize);
- }
-
- // Check it
- CheckValid();
-
- // Alert derived classes
- EncodedFilenameChanged();
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::WriteToStream(IOStream &)
-// Purpose: Writes the filename to a stream
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilename::WriteToStream(IOStream &rStream) const
-{
- CheckValid();
-
- rStream.Write(mEncryptedName.c_str(), mEncryptedName.size());
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::EncodedFilenameChanged()
-// Purpose: The encoded filename stored has changed
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilename::EncodedFilenameChanged()
-{
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::IsEncrypted()
-// Purpose: Returns true if the filename is stored using an encrypting encoding
-// Created: 1/12/03
-//
-// --------------------------------------------------------------------------
-bool BackupStoreFilename::IsEncrypted() const
-{
- return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) !=
- Encoding_Clear;
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilename::SetAsClearFilename(const char *)
-// Purpose: Sets this object to be a valid filename, but with a
-// filename in the clear. Used on the server to create
-// filenames when there's no way of encrypting it.
-// Created: 22/4/04
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilename::SetAsClearFilename(const char *Clear)
-{
- // Make std::string from the clear name
- std::string toEncode(Clear);
-
- // Make an encoded string
- char hdr[2];
- BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear);
- std::string encoded(hdr, 2);
- encoded += toEncode;
- ASSERT(encoded.size() == toEncode.size() + 2);
-
- // Store the encoded string
- mEncryptedName.assign(encoded);
-
- // Stuff which must be done
- EncodedFilenameChanged();
- CheckValid(false);
-}
-
-
-
Deleted: box/trunk/lib/backupclient/BackupStoreFilename.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFilename.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFilename.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,107 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFilename.h
-// Purpose: Filename for the backup store
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREFILENAME__H
-#define BACKUPSTOREFILENAME__H
-
-#include <string>
-
-class Protocol;
-class IOStream;
-
-// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
-// don't define this -- the problem of memory usage still appears without this.
-// It's just that this class really showed up the problem. Instead, malloc allocation
-// is globally defined in BoxPlatform.h, for troublesome libraries.
-
-#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
- // Use a malloc_allocated string, because the STL default allocators really screw up with
- // memory allocation, particularly with this class.
- // Makes a few things a bit messy and inefficient with conversions.
- // Given up using this, and use global malloc allocation instead, but thought it
- // worth leaving this code in just in case it's useful for the future.
- typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base;
- // If this is changed, change GetClearFilename() back to returning a reference.
-#else
- typedef std::string BackupStoreFilename_base;
-#endif
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupStoreFilename
-// Purpose: Filename for the backup store
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-class BackupStoreFilename /* : public BackupStoreFilename_base */
-{
-private:
- std::string mEncryptedName;
-
-public:
- BackupStoreFilename();
- BackupStoreFilename(const BackupStoreFilename &rToCopy);
- virtual ~BackupStoreFilename();
-
- bool CheckValid(bool ExceptionIfInvalid = true) const;
-
- void ReadFromProtocol(Protocol &rProtocol);
- void WriteToProtocol(Protocol &rProtocol) const;
-
- void ReadFromStream(IOStream &rStream, int Timeout);
- void WriteToStream(IOStream &rStream) const;
-
- void SetAsClearFilename(const char *Clear);
-
- // Check that it's encrypted
- bool IsEncrypted() const;
-
- // These enumerated types belong in the base class so
- // the CheckValid() function can make sure that the encoding
- // is a valid encoding
- enum
- {
- Encoding_Min = 1,
- Encoding_Clear = 1,
- Encoding_Blowfish = 2,
- Encoding_Max = 2
- };
-
- const std::string& GetEncodedFilename() const
- {
- return mEncryptedName;
- }
-
- bool operator==(const BackupStoreFilename& rOther) const
- {
- return mEncryptedName == rOther.mEncryptedName;
- }
-
- bool operator!=(const BackupStoreFilename& rOther) const
- {
- return mEncryptedName != rOther.mEncryptedName;
- }
-
-protected:
- virtual void EncodedFilenameChanged();
- void SetEncodedFilename(const std::string &rEncoded)
- {
- mEncryptedName = rEncoded;
- }
-};
-
-// On the wire utilities for class and derived class
-#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
-#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3)
-
-#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;}
-
-#endif // BACKUPSTOREFILENAME__H
-
Deleted: box/trunk/lib/backupclient/BackupStoreFilenameClear.cpp
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFilenameClear.cpp 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFilenameClear.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,335 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFilenameClear.cpp
-// Purpose: BackupStoreFilenames in the clear
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-
-#include "Box.h"
-#include "BackupStoreFilenameClear.h"
-#include "BackupStoreException.h"
-#include "CipherContext.h"
-#include "CipherBlowfish.h"
-#include "Guards.h"
-#include "Logging.h"
-
-#include "MemLeakFindOn.h"
-
-// Hide private variables from the rest of the world
-namespace
-{
- int sEncodeMethod = BackupStoreFilename::Encoding_Clear;
- CipherContext sBlowfishEncrypt;
- CipherContext sBlowfishDecrypt;
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::BackupStoreFilenameClear()
-// Purpose: Default constructor, creates an invalid filename
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilenameClear::BackupStoreFilenameClear()
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &)
-// Purpose: Creates a filename, encoding from the given string
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode)
-{
- SetClearFilename(rToEncode);
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &)
-// Purpose: Copy constructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy)
- : BackupStoreFilename(rToCopy),
- mClearFilename(rToCopy.mClearFilename)
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
-// Purpose: Copy from base class
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
- : BackupStoreFilename(rToCopy)
-{
- // Will get a clear filename when it's required
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear()
-// Purpose: Destructor
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-BackupStoreFilenameClear::~BackupStoreFilenameClear()
-{
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::GetClearFilename()
-// Purpose: Get the unencoded filename
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
-const std::string BackupStoreFilenameClear::GetClearFilename() const
-{
- MakeClearAvailable();
- // When modifying, remember to change back to reference return if at all possible
- // -- returns an object rather than a reference to allow easy use with other code.
- return std::string(mClearFilename.c_str(), mClearFilename.size());
-}
-#else
-const std::string &BackupStoreFilenameClear::GetClearFilename() const
-{
- MakeClearAvailable();
- return mClearFilename;
-}
-#endif
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &)
-// Purpose: Encode and make available the clear filename
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode)
-{
- // Only allow Blowfish encodings
- if(sEncodeMethod != Encoding_Blowfish)
- {
- THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup)
- }
-
- // Make an encoded string with blowfish encryption
- EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish);
-
- // Store the clear filename
- mClearFilename.assign(rToEncode.c_str(), rToEncode.size());
-
- // Make sure we did the right thing
- if(!CheckValid(false))
- {
- THROW_EXCEPTION(BackupStoreException, Internal)
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::MakeClearAvailable()
-// Purpose: Private. Make sure the clear filename is available
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::MakeClearAvailable() const
-{
- if(!mClearFilename.empty())
- return; // nothing to do
-
- // Check valid
- CheckValid();
-
- // Decode the header
- int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename());
- int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename());
-
- // Decode based on encoding given in the header
- switch(encoding)
- {
- case Encoding_Clear:
- BOX_TRACE("**** BackupStoreFilename encoded with "
- "Clear encoding ****");
- mClearFilename.assign(GetEncodedFilename().c_str() + 2,
- size - 2);
- break;
-
- case Encoding_Blowfish:
- DecryptEncoded(sBlowfishDecrypt);
- break;
-
- default:
- THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding)
- break;
- }
-}
-
-
-// Buffer for encoding and decoding -- do this all in one single buffer to
-// avoid lots of string allocation, which stuffs up memory usage.
-// These static memory vars are, of course, not thread safe, but we don't use threads.
-static int sEncDecBufferSize = 0;
-static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0;
-
-static void EnsureEncDecBufferSize(int BufSize)
-{
- if(spEncDecBuffer == 0)
- {
-#ifndef WIN32
- BOX_TRACE("Allocating filename encoding/decoding buffer "
- "with size " << BufSize);
-#endif
- spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize);
- MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer);
- MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer);
- sEncDecBufferSize = BufSize;
- }
- else
- {
- if(sEncDecBufferSize < BufSize)
- {
- BOX_TRACE("Reallocating filename encoding/decoding "
- "buffer from " << sEncDecBufferSize <<
- " to " << BufSize);
- spEncDecBuffer->Resize(BufSize);
- sEncDecBufferSize = BufSize;
- MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer);
- }
- }
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int)
-// Purpose: Private. Assigns the encoded filename string, encrypting.
-// Created: 1/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding)
-{
- // Work out max size
- int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4;
-
- // Make sure encode/decode buffer has enough space
- EnsureEncDecBufferSize(maxOutSize);
-
- // Pointer to buffer
- uint8_t *buffer = *spEncDecBuffer;
-
- // Encode -- do entire block in one go
- int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size());
- // and add in header size
- encSize += 2;
-
- // Adjust header
- BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding);
-
- // Store the encoded string
- SetEncodedFilename(std::string((char*)buffer, encSize));
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &)
-// Purpose: Decrypt the encoded filename using the cipher context
-// Created: 1/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const
-{
- const std::string& rEncoded = GetEncodedFilename();
-
- // Work out max size
- int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4;
-
- // Make sure encode/decode buffer has enough space
- EnsureEncDecBufferSize(maxOutSize);
-
- // Pointer to buffer
- uint8_t *buffer = *spEncDecBuffer;
-
- // Decrypt
- const char *str = rEncoded.c_str() + 2;
- int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2);
-
- // Assign to this
- mClearFilename.assign((char*)buffer, sizeOut);
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::EncodedFilenameChanged()
-// Purpose: The encoded filename stored has changed
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::EncodedFilenameChanged()
-{
- BackupStoreFilename::EncodedFilenameChanged();
-
- // Delete stored filename in clear
- mClearFilename.erase();
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int)
-// Purpose: Set the key used for Blowfish encryption of filenames
-// Created: 1/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength)
-{
- // Initialisation vector not used. Can't use a different vector for each filename as
- // that would stop comparisions on the server working.
- sBlowfishEncrypt.Reset();
- sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
- ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength);
- sBlowfishEncrypt.SetIV(pIV);
- sBlowfishDecrypt.Reset();
- sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
- ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength);
- sBlowfishDecrypt.SetIV(pIV);
-}
-
-// --------------------------------------------------------------------------
-//
-// Function
-// Name: BackupStoreFilenameClear::SetEncodingMethod(int)
-// Purpose: Set the encoding method used for filenames
-// Created: 1/12/03
-//
-// --------------------------------------------------------------------------
-void BackupStoreFilenameClear::SetEncodingMethod(int Method)
-{
- sEncodeMethod = Method;
-}
-
-
-
Deleted: box/trunk/lib/backupclient/BackupStoreFilenameClear.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreFilenameClear.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreFilenameClear.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,60 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreFilenameClear.h
-// Purpose: BackupStoreFilenames in the clear
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREFILENAMECLEAR__H
-#define BACKUPSTOREFILENAMECLEAR__H
-
-#include "BackupStoreFilename.h"
-
-class CipherContext;
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: BackupStoreFilenameClear
-// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version
-// Created: 2003/08/26
-//
-// --------------------------------------------------------------------------
-class BackupStoreFilenameClear : public BackupStoreFilename
-{
-public:
- BackupStoreFilenameClear();
- BackupStoreFilenameClear(const std::string &rToEncode);
- BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy);
- BackupStoreFilenameClear(const BackupStoreFilename &rToCopy);
- virtual ~BackupStoreFilenameClear();
-
- // Because we need to use a different allocator for this class to avoid
- // nasty things happening, can't return this as a reference. Which is a
- // pity. But probably not too bad.
-#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
- const std::string GetClearFilename() const;
-#else
- const std::string &GetClearFilename() const;
-#endif
- void SetClearFilename(const std::string &rToEncode);
-
- // Setup for encryption of filenames
- static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength);
- static void SetEncodingMethod(int Method);
-
-protected:
- void MakeClearAvailable() const;
- virtual void EncodedFilenameChanged();
- void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding);
- void DecryptEncoded(CipherContext &rCipherContext) const;
-
-private:
- mutable BackupStoreFilename_base mClearFilename;
-};
-
-#endif // BACKUPSTOREFILENAMECLEAR__H
-
-
Deleted: box/trunk/lib/backupclient/BackupStoreObjectMagic.h
===================================================================
--- box/trunk/lib/backupclient/BackupStoreObjectMagic.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/BackupStoreObjectMagic.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,31 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: BackupStoreObjectMagic.h
-// Purpose: Magic values for the start of objects in the backup store
-// Created: 19/11/03
-//
-// --------------------------------------------------------------------------
-
-#ifndef BACKUPSTOREOBJECTMAGIC__H
-#define BACKUPSTOREOBJECTMAGIC__H
-
-// Each of these values is the first 4 bytes of the object file.
-// Remember to swap from network to host byte order.
-
-// Magic value for file streams
-#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65
-// Do not use v0 in any new code!
-#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45
-
-// Magic for the block index at the file stream -- used to
-// ensure streams are reordered as expected
-#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478
-// Do not use v0 in any new code!
-#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B
-
-// Magic value for directory streams
-#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F
-
-#endif // BACKUPSTOREOBJECTMAGIC__H
-
Deleted: box/trunk/lib/backupclient/Makefile.extra
===================================================================
--- box/trunk/lib/backupclient/Makefile.extra 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/Makefile.extra 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,16 +0,0 @@
-
-MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
-
-GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt
-
-# AUTOGEN SEEDING
-autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt
- $(_PERL) $(GEN_CMD_SRV)
-
-
-MAKEEXCEPTION = ../../lib/common/makeexception.pl
-
-# AUTOGEN SEEDING
-autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt
- $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt
-
Deleted: box/trunk/lib/backupclient/RunStatusProvider.h
===================================================================
--- box/trunk/lib/backupclient/RunStatusProvider.h 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/lib/backupclient/RunStatusProvider.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -1,29 +0,0 @@
-// --------------------------------------------------------------------------
-//
-// File
-// Name: RunStatusProvider.h
-// Purpose: Declares the RunStatusProvider interface.
-// Created: 2008/08/14
-//
-// --------------------------------------------------------------------------
-
-#ifndef RUNSTATUSPROVIDER__H
-#define RUNSTATUSPROVIDER__H
-
-// --------------------------------------------------------------------------
-//
-// Class
-// Name: RunStatusProvider
-// Purpose: Provides a StopRun() method which returns true if
-// the current backup should be halted.
-// Created: 2005/11/15
-//
-// --------------------------------------------------------------------------
-class RunStatusProvider
-{
- public:
- virtual ~RunStatusProvider() { }
- virtual bool StopRun() = 0;
-};
-
-#endif // RUNSTATUSPROVIDER__H
Copied: box/trunk/lib/backupstore/BackupClientFileAttributes.cpp (from rev 2943, box/trunk/lib/backupclient/BackupClientFileAttributes.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupClientFileAttributes.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupClientFileAttributes.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,1188 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.cpp
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <limits.h>
+
+#include <algorithm>
+#include <cstring>
+#include <new>
+#include <vector>
+
+#ifdef HAVE_SYS_XATTR_H
+#include <cerrno>
+#include <sys/xattr.h>
+#endif
+
+#include <cstring>
+
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+#include "BoxTimeToUnix.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "MD5Digest.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#define ATTRIBUTETYPE_GENERIC_UNIX 1
+
+#define ATTRIBUTE_ENCODING_BLOWFISH 2
+
+typedef struct
+{
+ int32_t AttributeType;
+ u_int32_t UID;
+ u_int32_t GID;
+ u_int64_t ModificationTime;
+ u_int64_t AttrModificationTime;
+ u_int32_t UserDefinedFlags;
+ u_int32_t FileGenerationNumber;
+ u_int16_t Mode;
+ // Symbolic link filename may follow
+ // Extended attribute (xattr) information may follow, format is:
+ // u_int32_t Size of extended attribute block (excluding this word)
+ // For each of NumberOfAttributes (sorted by AttributeName):
+ // u_int16_t AttributeNameLength
+ // char AttributeName[AttributeNameLength]
+ // u_int32_t AttributeValueLength
+ // unsigned char AttributeValue[AttributeValueLength]
+ // AttributeName is 0 terminated, AttributeValue is not (and may be binary data)
+} attr_StreamFormat;
+
+// This has wire packing so it's compatible across platforms
+// Use wider than necessary sizes, just to be careful.
+typedef struct
+{
+ int32_t uid, gid, mode;
+ #ifdef WIN32
+ int64_t fileCreationTime;
+ #endif
+} attributeHashData;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256
+
+// Hide private static variables from the rest of the world
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace
+{
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+ uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH];
+ int sAttributeHashSecretLength = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes()
+// Purpose: Default constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes()
+ : mpClearAttributes(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &)
+// Purpose: Copy constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::~BackupClientFileAttributes()
+// Purpose: Destructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::~BackupClientFileAttributes()
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ mpClearAttributes = 0;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &)
+// Purpose: Assignment operator
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+// Assume users play nice
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &)
+// Purpose: Comparison operator
+// Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ return mpClearAttributes->operator==(*rAttr.mpClearAttributes);
+}
+// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data.
+/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const
+{
+ StreamableMemBlock *pDecoded = 0;
+
+ try
+ {
+ EnsureClearAvailable();
+ StreamableMemBlock *pDecoded = MakeClear(rAttr);
+
+ // Compare using clear version
+ bool compared = mpClearAttributes->operator==(rAttr);
+
+ // Delete temporary
+ delete pDecoded;
+
+ return compared;
+ }
+ catch(...)
+ {
+ delete pDecoded;
+ throw;
+ }
+}*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
+// Purpose: Compare, optionally ignoring the attribute
+// modification time and/or modification time, and some
+// data which is irrelevant in practise (eg file
+// generation number)
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr,
+ bool IgnoreAttrModTime, bool IgnoreModTime) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ // Check sizes are the same, as a first check
+ if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
+ {
+ BOX_TRACE("Attribute Compare: Attributes objects are "
+ "different sizes, cannot compare them: local " <<
+ mpClearAttributes->GetSize() << " bytes, remote " <<
+ rAttr.mpClearAttributes->GetSize() << " bytes");
+ return false;
+ }
+
+ // Then check the elements of the two things
+ // Bytes are checked in network order, but this doesn't matter as we're only checking for equality.
+ attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
+
+ #define COMPARE(attribute, message) \
+ if (a1->attribute != a2->attribute) \
+ { \
+ BOX_TRACE("Attribute Compare: " << message << " differ: " \
+ "local " << ntoh(a1->attribute) << ", " \
+ "remote " << ntoh(a2->attribute)); \
+ return false; \
+ }
+ COMPARE(AttributeType, "Attribute types");
+ COMPARE(UID, "UIDs");
+ COMPARE(GID, "GIDs");
+ COMPARE(UserDefinedFlags, "User-defined flags");
+ COMPARE(Mode, "Modes");
+
+ if(!IgnoreModTime)
+ {
+ uint64_t t1 = box_ntoh64(a1->ModificationTime);
+ uint64_t t2 = box_ntoh64(a2->ModificationTime);
+ time_t s1 = BoxTimeToSeconds(t1);
+ time_t s2 = BoxTimeToSeconds(t2);
+ if(s1 != s2)
+ {
+ BOX_TRACE("Attribute Compare: File modification "
+ "times differ: local " <<
+ FormatTime(t1, true) << " (" << s1 << "), "
+ "remote " <<
+ FormatTime(t2, true) << " (" << s2 << ")");
+ return false;
+ }
+ }
+
+ if(!IgnoreAttrModTime)
+ {
+ uint64_t t1 = box_ntoh64(a1->AttrModificationTime);
+ uint64_t t2 = box_ntoh64(a2->AttrModificationTime);
+ time_t s1 = BoxTimeToSeconds(t1);
+ time_t s2 = BoxTimeToSeconds(t2);
+ if(s1 != s2)
+ {
+ BOX_TRACE("Attribute Compare: Attribute modification "
+ "times differ: local " <<
+ FormatTime(t1, true) << " (" << s1 << "), "
+ "remote " <<
+ FormatTime(t2, true) << " (" << s2 << ")");
+ return false;
+ }
+ }
+
+ // Check symlink string?
+ unsigned int size = mpClearAttributes->GetSize();
+ if(size > sizeof(attr_StreamFormat))
+ {
+ // Symlink strings don't match. This also compares xattrs
+ int datalen = size - sizeof(attr_StreamFormat);
+
+ if(::memcmp(a1 + 1, a2 + 1, datalen) != 0)
+ {
+ std::string s1((char *)(a1 + 1), datalen);
+ std::string s2((char *)(a2 + 1), datalen);
+ BOX_TRACE("Attribute Compare: Symbolic link target "
+ "or extended attributes differ: "
+ "local " << PrintEscapedBinaryData(s1) << ", "
+ "remote " << PrintEscapedBinaryData(s2));
+ return false;
+ }
+ }
+
+ // Passes all test, must be OK
+ return true;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributes(
+// const char *Filename, bool ZeroModificationTimes,
+// box_time_t *pModTime, box_time_t *pAttrModTime,
+// int64_t *pFileSize, InodeRefType *pInodeNumber,
+// bool *pHasMultipleLinks)
+// Purpose: Read the attributes of the file, and store them
+// ready for streaming. Optionally retrieve the
+// modification time and attribute modification time.
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributes(const char *Filename,
+ bool ZeroModificationTimes, box_time_t *pModTime,
+ box_time_t *pAttrModTime, int64_t *pFileSize,
+ InodeRefType *pInodeNumber, bool *pHasMultipleLinks)
+{
+ StreamableMemBlock *pnewAttr = 0;
+ try
+ {
+ EMU_STRUCT_STAT st;
+ if(EMU_LSTAT(Filename, &st) != 0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to stat file: '" <<
+ Filename << "'");
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Modification times etc
+ if(pModTime) {*pModTime = FileModificationTime(st);}
+ if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);}
+ if(pFileSize) {*pFileSize = st.st_size;}
+ if(pInodeNumber) {*pInodeNumber = st.st_ino;}
+ if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);}
+
+ pnewAttr = new StreamableMemBlock;
+
+ FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes);
+
+#ifndef WIN32
+ // Is it a link?
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ FillAttributesLink(*pnewAttr, Filename, st);
+ }
+#endif
+
+ FillExtendedAttr(*pnewAttr, Filename);
+
+#ifdef WIN32
+ //this is to catch those problems with invalid time stamps stored...
+ //need to find out the reason why - but also a catch as well.
+
+ attr_StreamFormat *pattr =
+ (attr_StreamFormat*)pnewAttr->GetBuffer();
+ ASSERT(pattr != 0);
+
+ // __time64_t winTime = BoxTimeToSeconds(
+ // pnewAttr->ModificationTime);
+
+ u_int64_t modTime = box_ntoh64(pattr->ModificationTime);
+ box_time_t modSecs = BoxTimeToSeconds(modTime);
+ __time64_t winTime = modSecs;
+
+ // _MAX__TIME64_T doesn't seem to be defined, but the code below
+ // will throw an assertion failure if we exceed it :-)
+ // Microsoft says dates up to the year 3000 are valid, which
+ // is a bit more than 15 * 2^32. Even that doesn't seem
+ // to be true (still aborts), but it can at least hold 2^32.
+ if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0)
+ {
+ BOX_ERROR("Invalid Modification Time caught for "
+ "file: '" << Filename << "'");
+ pattr->ModificationTime = 0;
+ }
+
+ modTime = box_ntoh64(pattr->AttrModificationTime);
+ modSecs = BoxTimeToSeconds(modTime);
+ winTime = modSecs;
+
+ if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0)
+ {
+ BOX_ERROR("Invalid Attribute Modification Time "
+ "caught for file: '" << Filename << "'");
+ pattr->AttrModificationTime = 0;
+ }
+#endif
+
+ // Attributes ready. Encrypt into this block
+ EncryptAttr(*pnewAttr);
+
+ // Store the new attributes
+ RemoveClear();
+ mpClearAttributes = pnewAttr;
+ pnewAttr = 0;
+ }
+ catch(...)
+ {
+ // clean up
+ delete pnewAttr;
+ pnewAttr = 0;
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributesLink()
+// Purpose: Private function, handles standard attributes for all objects
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes)
+{
+ outputBlock.ResizeBlock(sizeof(attr_StreamFormat));
+ attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer();
+ ASSERT(pattr != 0);
+
+ // Fill in the entries
+ pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX);
+ pattr->UID = htonl(st.st_uid);
+ pattr->GID = htonl(st.st_gid);
+ if(ZeroModificationTimes)
+ {
+ pattr->ModificationTime = 0;
+ pattr->AttrModificationTime = 0;
+ }
+ else
+ {
+ pattr->ModificationTime = box_hton64(FileModificationTime(st));
+ pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st));
+ }
+ pattr->Mode = htons(st.st_mode);
+
+#ifndef HAVE_STRUCT_STAT_ST_FLAGS
+ pattr->UserDefinedFlags = 0;
+ pattr->FileGenerationNumber = 0;
+#else
+ pattr->UserDefinedFlags = htonl(st.st_flags);
+ pattr->FileGenerationNumber = htonl(st.st_gen);
+#endif
+}
+#ifndef WIN32
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributesLink()
+// Purpose: Private function, handles the case where a symbolic link is needed
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st)
+{
+ // Make sure we're only called for symbolic links
+ ASSERT((st.st_mode & S_IFMT) == S_IFLNK);
+
+ // Get the filename the link is linked to
+ char linkedTo[PATH_MAX+4];
+ int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX);
+ if(linkedToSize == -1)
+ {
+ BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'");
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+
+ int oldSize = outputBlock.GetSize();
+ outputBlock.ResizeBlock(oldSize+linkedToSize+1);
+ char* buffer = static_cast<char*>(outputBlock.GetBuffer());
+
+ // Add the path name for the symbolic link, and add 0 termination
+ std::memcpy(buffer+oldSize, linkedTo, linkedToSize);
+ buffer[oldSize+linkedToSize] = '\0';
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**)
+// Purpose: Private function, read the extended attributes of the file into the block
+// Created: 2005/06/12
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename)
+{
+#ifdef HAVE_SYS_XATTR_H
+ int listBufferSize = 10000;
+ char* list = new char[listBufferSize];
+
+ try
+ {
+ // This returns an unordered list of attribute names, each 0 terminated,
+ // concatenated together
+ int listSize = ::llistxattr(Filename, list, listBufferSize);
+
+ if(listSize>listBufferSize)
+ {
+ delete[] list, list = NULL;
+ list = new char[listSize];
+ listSize = ::llistxattr(Filename, list, listSize);
+ }
+
+ if(listSize>0)
+ {
+ // Extract list of attribute names so we can sort them
+ std::vector<std::string> attrKeys;
+ for(int i = 0; i<listSize; ++i)
+ {
+ std::string attrKey(list+i);
+ i += attrKey.size();
+ attrKeys.push_back(attrKey);
+ }
+ sort(attrKeys.begin(), attrKeys.end());
+
+ // Make initial space in block
+ int xattrSize = outputBlock.GetSize();
+ int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000;
+ outputBlock.ResizeBlock(xattrBufferSize);
+ unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
+
+ // Leave space for attr block size later
+ int xattrBlockSizeOffset = xattrSize;
+ xattrSize += sizeof(u_int32_t);
+
+ // Loop for each attribute
+ for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI)
+ {
+ std::string attrKey(*attrKeyI);
+
+ if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize))
+ {
+ xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2;
+ outputBlock.ResizeBlock(xattrBufferSize);
+ buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
+ }
+
+ // Store length and text for attibute name
+ u_int16_t keyLength = htons(attrKey.size()+1);
+ std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t));
+ xattrSize += sizeof(u_int16_t);
+ std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1);
+ xattrSize += attrKey.size()+1;
+
+ // Leave space for value size
+ int valueSizeOffset = xattrSize;
+ xattrSize += sizeof(u_int32_t);
+
+ // Find size of attribute (must call with buffer and length 0 on some platforms,
+ // as -1 is returned if the data doesn't fit.)
+ int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0);
+ if(valueSize<0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to get "
+ "extended attribute size of "
+ "'" << Filename << "': " <<
+ attrKey);
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+
+ // Resize block, if needed
+ if(xattrSize+valueSize>xattrBufferSize)
+ {
+ xattrBufferSize = (xattrBufferSize+valueSize)*2;
+ outputBlock.ResizeBlock(xattrBufferSize);
+ buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
+ }
+
+ // This gets the attribute value (may be text or binary), no termination
+ valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize);
+ if(valueSize<0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to get "
+ "extended attribute of "
+ "'" << Filename << "': " <<
+ attrKey);
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ xattrSize += valueSize;
+
+ // Fill in value size
+ u_int32_t valueLength = htonl(valueSize);
+ std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t));
+ }
+
+ // Fill in attribute block size
+ u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t));
+ std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t));
+
+ outputBlock.ResizeBlock(xattrSize);
+ }
+ else if(listSize<0)
+ {
+ if(errno == EOPNOTSUPP || errno == EACCES)
+ {
+ // fail silently
+ }
+ else if(errno == ERANGE)
+ {
+ BOX_ERROR("Failed to list extended "
+ "attributes of '" << Filename << "': "
+ "buffer too small, not backed up");
+ }
+ else
+ {
+ BOX_LOG_SYS_ERROR("Failed to list extended "
+ "attributes of '" << Filename << "', "
+ "not backed up");
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ }
+ }
+ catch(...)
+ {
+ delete[] list;
+ throw;
+ }
+ delete[] list;
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::GetModificationTimes()
+// Purpose: Returns the modification time embedded in the
+// attributes.
+// Created: 2010/02/24
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::GetModificationTimes(
+ box_time_t *pModificationTime,
+ box_time_t *pAttrModificationTime) const
+{
+ // Got something loaded
+ if(GetSize() <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make sure there are clear attributes to use
+ EnsureClearAvailable();
+ ASSERT(mpClearAttributes != 0);
+
+ // Check if the decrypted attributes are small enough, and the type of attributes stored
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+ {
+ // Don't know what to do with these
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+
+ // Check there is enough space for an attributes block
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get pointer to structure
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+
+ if(pModificationTime)
+ {
+ *pModificationTime = box_ntoh64(pattr->ModificationTime);
+ }
+
+ if(pAttrModificationTime)
+ {
+ *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::WriteAttributes(const char *)
+// Purpose: Apply the stored attributes to the file
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteAttributes(const char *Filename,
+ bool MakeUserWritable) const
+{
+ // Got something loaded
+ if(GetSize() <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make sure there are clear attributes to use
+ EnsureClearAvailable();
+ ASSERT(mpClearAttributes != 0);
+
+ // Check if the decrypted attributes are small enough, and the type of attributes stored
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+ {
+ // Don't know what to do with these
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+
+ // Check there is enough space for an attributes block
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get pointer to structure
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ int xattrOffset = sizeof(attr_StreamFormat);
+
+ // is it a symlink?
+ int16_t mode = ntohs(pattr->Mode);
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ // Check things are sensible
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1)
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+#ifdef WIN32
+ BOX_WARNING("Cannot create symbolic links on Windows: '" <<
+ Filename << "'");
+#else
+ // Make a symlink, first deleting anything in the way
+ ::unlink(Filename);
+ if(::symlink((char*)(pattr + 1), Filename) != 0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename <<
+ "' to '" << (char*)(pattr + 1) << "'");
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+#endif
+
+ xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1;
+ }
+
+ // If working as root, set user IDs
+ if(::geteuid() == 0)
+ {
+ #ifndef HAVE_LCHOWN
+ // only if not a link, can't set their owner on this platform
+ if((mode & S_IFMT) != S_IFLNK)
+ {
+ // Not a link, use normal chown
+ if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to change "
+ "owner of file "
+ "'" << Filename << "'");
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+ #else
+ // use the version which sets things on symlinks
+ if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to change owner of "
+ "symbolic link '" << Filename << "'");
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ #endif
+ }
+
+ if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize())
+ {
+ WriteExtendedAttr(Filename, xattrOffset);
+ }
+
+ // Stop now if symlink, because otherwise it'll just be applied to the target
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ return;
+ }
+
+ // Set modification time?
+ box_time_t modtime = box_ntoh64(pattr->ModificationTime);
+ if(modtime != 0)
+ {
+ // Work out times as timevals
+ struct timeval times[2];
+
+ #ifdef WIN32
+ BoxTimeToTimeval(box_ntoh64(pattr->ModificationTime),
+ times[1]);
+ BoxTimeToTimeval(box_ntoh64(pattr->AttrModificationTime),
+ times[0]);
+ // Because stat() returns the creation time in the ctime
+ // field under Windows, and this gets saved in the
+ // AttrModificationTime field of the serialised attributes,
+ // we subvert the first parameter of emu_utimes() to allow
+ // it to be reset to the right value on the restored file.
+ #else
+ BoxTimeToTimeval(modtime, times[1]);
+ // Copy access time as well, why not, got to set it to something
+ times[0] = times[1];
+ // Attr modification time will be changed anyway,
+ // nothing that can be done about it
+ #endif
+
+ // Try to apply
+ if(::utimes(Filename, times) != 0)
+ {
+ BOX_LOG_SYS_WARNING("Failed to change times of "
+ "file '" << Filename << "' to ctime=" <<
+ BOX_FORMAT_TIMESPEC(times[0]) << ", mtime=" <<
+ BOX_FORMAT_TIMESPEC(times[1]));
+ }
+ }
+
+ if (MakeUserWritable)
+ {
+ mode |= S_IRWXU;
+ }
+
+ // Apply everything else... (allowable mode flags only)
+ // Mode must be done last (think setuid)
+ if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID
+ | S_ISGID | S_ISVTX)) != 0)
+ {
+ BOX_LOG_SYS_ERROR("Failed to change permissions of file "
+ "'" << Filename << "'");
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::IsSymLink()
+// Purpose: Do these attributes represent a symbolic link?
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::IsSymLink() const
+{
+ EnsureClearAvailable();
+
+ // Got the right kind of thing?
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get the type of attributes stored
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat))
+ {
+ // Check link
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK;
+ }
+
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::RemoveClear()
+// Purpose: Private. Deletes any clear version of the attributes that may be held
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::RemoveClear() const
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ }
+ mpClearAttributes = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EnsureClearAvailable()
+// Purpose: Private. Makes sure the clear version is available
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EnsureClearAvailable() const
+{
+ if(mpClearAttributes == 0)
+ {
+ mpClearAttributes = MakeClear(*this);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset)
+// Purpose: Private function, apply the stored extended attributes to the file
+// Created: 2005/06/13
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const
+{
+#ifdef HAVE_SYS_XATTR_H
+ const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer());
+
+ u_int32_t xattrBlockLength = 0;
+ std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t));
+ int xattrBlockSize = ntohl(xattrBlockLength);
+ xattrOffset += sizeof(u_int32_t);
+
+ int xattrEnd = xattrOffset+xattrBlockSize;
+ if(xattrEnd>mpClearAttributes->GetSize())
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ while(xattrOffset<xattrEnd)
+ {
+ u_int16_t keyLength = 0;
+ std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t));
+ int keySize = ntohs(keyLength);
+ xattrOffset += sizeof(u_int16_t);
+
+ const char* key = buffer+xattrOffset;
+ xattrOffset += keySize;
+
+ u_int32_t valueLength = 0;
+ std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t));
+ int valueSize = ntohl(valueLength);
+ xattrOffset += sizeof(u_int32_t);
+
+ // FIXME: Warn on EOPNOTSUPP
+ if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP)
+ {
+ BOX_LOG_SYS_ERROR("Failed to set extended attributes "
+ "on file '" << Filename << "'");
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+
+ xattrOffset += valueSize;
+ }
+
+ ASSERT(xattrOffset==xattrEnd);
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &)
+// Purpose: Static. Decrypts stored attributes.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted)
+{
+ // New block
+ StreamableMemBlock *pdecrypted = 0;
+
+ try
+ {
+ // Check the block is big enough for IV and header
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+ if(rEncrypted.GetSize() <= (ivSize + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes);
+ }
+
+ // How much space is needed for the output?
+ int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize);
+
+ // Allocate it
+ pdecrypted = new StreamableMemBlock(maxDecryptedSize);
+
+ // ptr to block
+ uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer();
+
+ // Check that the header has right type
+ if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH)
+ {
+ THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding);
+ }
+
+ // Set IV
+ sBlowfishDecrypt.SetIV(encBlock + 1);
+
+ // Decrypt
+ int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1));
+
+ // Resize block to fit
+ pdecrypted->ResizeBlock(decryptedSize);
+ }
+ catch(...)
+ {
+ delete pdecrypted;
+ pdecrypted = 0;
+ }
+
+ return pdecrypted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &)
+// Purpose: Private. Encrypt the given attributes into this block.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt)
+{
+ // Free any existing block
+ FreeBlock();
+
+ // Work out the maximum amount of space we need
+ int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize());
+ // And the size of the IV
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+
+ // Allocate this space
+ AllocateBlock(maxEncryptedSize + ivSize + 1);
+
+ // Store the encoding byte
+ uint8_t *block = (uint8_t*)GetBuffer();
+ block[0] = ATTRIBUTE_ENCODING_BLOWFISH;
+
+ // Generate and store an IV for this attribute block
+ int ivSize2 = 0;
+ const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2);
+ ASSERT(ivSize == ivSize2);
+
+ // Copy into the encrypted block
+ ::memcpy(block + 1, iv, ivSize);
+
+ // Do the transform
+ int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize());
+
+ // Resize this block
+ ResizeBlock(encrytedSize + ivSize + 1);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int)
+// Purpose: Set the secret for the filename attribute hash
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength)
+{
+ if(SecretLength > (int)sizeof(sAttributeHashSecret))
+ {
+ SecretLength = sizeof(sAttributeHashSecret);
+ }
+ if(SecretLength < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Copy
+ ::memcpy(sAttributeHashSecret, pSecret, SecretLength);
+ sAttributeHashSecretLength = SecretLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::GenerateAttributeHash(
+// struct stat &, const std::string &,
+// const std::string &)
+// Purpose: Generate a 64 bit hash from the attributes, used to
+// detect changes. Include filename in the hash, so
+// that it changes from one file to another, so don't
+// reveal identical attributes.
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st,
+ const std::string &filename, const std::string &leafname)
+{
+ if(sAttributeHashSecretLength == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet)
+ }
+
+ // Assemble stuff we're interested in
+ attributeHashData hashData;
+ memset(&hashData, 0, sizeof(hashData));
+ // Use network byte order and large sizes to be cross platform
+ hashData.uid = htonl(st.st_uid);
+ hashData.gid = htonl(st.st_gid);
+ hashData.mode = htonl(st.st_mode);
+
+ #ifdef WIN32
+ // On Windows, the "file attribute modification time" is the
+ // file creation time, and we want to back this up, restore
+ // it and compare it.
+ //
+ // On other platforms, it's not very important and can't
+ // reliably be set to anything other than the current time.
+ hashData.fileCreationTime = box_hton64(st.st_ctime);
+ #endif
+
+ StreamableMemBlock xattr;
+ FillExtendedAttr(xattr, filename.c_str());
+
+ // Create a MD5 hash of the data, filename, and secret
+ MD5Digest digest;
+ digest.Add(&hashData, sizeof(hashData));
+ digest.Add(xattr.GetBuffer(), xattr.GetSize());
+ digest.Add(leafname.c_str(), leafname.size());
+ digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
+ digest.Finish();
+
+ // Return the first 64 bits of the hash
+ uint64_t result = *((uint64_t *)(digest.DigestAsData()));
+ return result;
+}
Copied: box/trunk/lib/backupstore/BackupClientFileAttributes.h (from rev 2943, box/trunk/lib/backupclient/BackupClientFileAttributes.h)
===================================================================
--- box/trunk/lib/backupstore/BackupClientFileAttributes.h (rev 0)
+++ box/trunk/lib/backupstore/BackupClientFileAttributes.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,78 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.h
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTFILEATTRIBUTES__H
+#define BACKUPCLIENTFILEATTRIBUTES__H
+
+#include <string>
+
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+EMU_STRUCT_STAT; // declaration
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupClientFileAttributes
+// Purpose: Storage, streaming and application of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+class BackupClientFileAttributes : public StreamableMemBlock
+{
+public:
+ BackupClientFileAttributes();
+ BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy);
+ BackupClientFileAttributes(const StreamableMemBlock &rToCopy);
+ ~BackupClientFileAttributes();
+ BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr);
+ BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr);
+ bool operator==(const BackupClientFileAttributes &rAttr) const;
+// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous?
+
+ bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const;
+
+ // Prevent access to base class members accidently
+ void Set();
+
+ void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false,
+ box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0,
+ InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0);
+ void WriteAttributes(const char *Filename,
+ bool MakeUserWritable = false) const;
+ void GetModificationTimes(box_time_t *pModificationTime,
+ box_time_t *pAttrModificationTime) const;
+
+ bool IsSymLink() const;
+
+ static void SetBlowfishKey(const void *pKey, int KeyLength);
+ static void SetAttributeHashSecret(const void *pSecret, int SecretLength);
+
+ static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname);
+ static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename);
+
+private:
+ static void FillAttributes(StreamableMemBlock &outputBlock,
+ const char *Filename, EMU_STRUCT_STAT &st,
+ bool ZeroModificationTimes);
+ static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st);
+ void WriteExtendedAttr(const char *Filename, int xattrOffset) const;
+
+ void RemoveClear() const;
+ void EnsureClearAvailable() const;
+ static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted);
+ void EncryptAttr(const StreamableMemBlock &rToEncrypt);
+
+private:
+ mutable StreamableMemBlock *mpClearAttributes;
+};
+
+#endif // BACKUPCLIENTFILEATTRIBUTES__H
+
Copied: box/trunk/lib/backupstore/BackupCommands.cpp (from rev 2943, box/trunk/bin/bbstored/BackupCommands.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupCommands.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupCommands.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,959 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupCommands.cpp
+// Purpose: Implement commands for the Backup store protocol
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <set>
+#include <sstream>
+
+#include "autogen_BackupProtocolServer.h"
+#include "autogen_RaidFileException.h"
+#include "BackupConstants.h"
+#include "BackupStoreContext.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreInfo.h"
+#include "BufferedStream.h"
+#include "CollectInBufferStream.h"
+#include "FileStream.h"
+#include "InvisibleTempFileStream.h"
+#include "RaidFileController.h"
+#include "StreamableMemBlock.h"
+
+#include "MemLeakFindOn.h"
+
+#define PROTOCOL_ERROR(code) \
+ std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( \
+ BackupProtocolServerError::ErrorType, \
+ BackupProtocolServerError::code));
+
+#define CHECK_PHASE(phase) \
+ if(rContext.GetPhase() != BackupStoreContext::phase) \
+ { \
+ return PROTOCOL_ERROR(Err_NotInRightProtocolPhase); \
+ }
+
+#define CHECK_WRITEABLE_SESSION \
+ if(rContext.SessionIsReadOnly()) \
+ { \
+ return PROTOCOL_ERROR(Err_SessionReadOnly); \
+ }
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Return the current version, or an error if the requested version isn't allowed
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Version)
+
+ // Correct version?
+ if(mVersion != BACKUP_STORE_SERVER_VERSION)
+ {
+ return PROTOCOL_ERROR(Err_WrongVersion);
+ }
+
+ // Mark the next phase
+ rContext.SetPhase(BackupStoreContext::Phase_Login);
+
+ // Return our version
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Return the current version, or an error if the requested version isn't allowed
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Login)
+
+ // Check given client ID against the ID in the certificate certificate
+ // and that the client actually has an account on this machine
+ if(mClientID != rContext.GetClientID())
+ {
+ BOX_WARNING("Failed login from client ID " <<
+ BOX_FORMAT_ACCOUNT(mClientID) <<
+ ": wrong certificate for this account");
+ return PROTOCOL_ERROR(Err_BadLogin);
+ }
+
+ if(!rContext.GetClientHasAccount())
+ {
+ BOX_WARNING("Failed login from client ID " <<
+ BOX_FORMAT_ACCOUNT(mClientID) <<
+ ": no such account on this server");
+ return PROTOCOL_ERROR(Err_BadLogin);
+ }
+
+ // If we need to write, check that nothing else has got a write lock
+ if((mFlags & Flags_ReadOnly) != Flags_ReadOnly)
+ {
+ // See if the context will get the lock
+ if(!rContext.AttemptToGetWriteLock())
+ {
+ BOX_WARNING("Failed to get write lock for Client ID " <<
+ BOX_FORMAT_ACCOUNT(mClientID));
+ return PROTOCOL_ERROR(Err_CannotLockStoreForWriting);
+ }
+
+ // Debug: check we got the lock
+ ASSERT(!rContext.SessionIsReadOnly());
+ }
+
+ // Load the store info
+ rContext.LoadStoreInfo();
+
+ // Get the last client store marker
+ int64_t clientStoreMarker = rContext.GetClientStoreMarker();
+
+ // Mark the next phase
+ rContext.SetPhase(BackupStoreContext::Phase_Commands);
+
+ // Log login
+ BOX_NOTICE("Login from Client ID " <<
+ BOX_FORMAT_ACCOUNT(mClientID) <<
+ " " <<
+ (((mFlags & Flags_ReadOnly) != Flags_ReadOnly)
+ ?"Read/Write":"Read-only"));
+
+ // Get the usage info for reporting to the client
+ int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0;
+ rContext.GetStoreDiscUsageInfo(blocksUsed, blocksSoftLimit, blocksHardLimit);
+
+ // Return success
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerLoginConfirmed(clientStoreMarker, blocksUsed, blocksSoftLimit, blocksHardLimit));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Marks end of conversation (Protocol framework handles this)
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ BOX_NOTICE("Session finished for Client ID " <<
+ BOX_FORMAT_ACCOUNT(rContext.GetClientID()));
+
+ // Let the context know about it
+ rContext.ReceivedFinishCommand();
+
+ // can be called in any phase
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerFinished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Command to list a directory
+// Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Store the listing to a stream
+ std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
+
+ try
+ {
+ // Ask the context for a directory
+ const BackupStoreDirectory &rdir(
+ rContext.GetDirectory(mObjectID));
+ rdir.WriteToStream(*stream, mFlagsMustBeSet,
+ mFlagsNotToBeSet, mSendAttributes,
+ false /* never send dependency info to the client */);
+ }
+ catch (RaidFileException &e)
+ {
+ if (e.GetSubType() == RaidFileException::RaidFileDoesntExist)
+ {
+ return PROTOCOL_ERROR(Err_DoesNotExist);
+ }
+ throw;
+ }
+
+ stream->SetForReading();
+
+ // Get the protocol to send the stream
+ rProtocol.SendStreamAfterCommand(stream.release());
+
+ return std::auto_ptr<ProtocolObject>(
+ new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Command to store a file on the server
+// Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ std::auto_ptr<ProtocolObject> hookResult =
+ rContext.StartCommandHook(*this);
+ if(hookResult.get())
+ {
+ return hookResult;
+ }
+
+ // Check that the diff from file actually exists, if it's specified
+ if(mDiffFromFileID != 0)
+ {
+ if(!rContext.ObjectExists(mDiffFromFileID,
+ BackupStoreContext::ObjectExists_File))
+ {
+ return PROTOCOL_ERROR(Err_DiffFromFileDoesNotExist);
+ }
+ }
+
+ // A stream follows, which contains the file
+ std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream());
+
+ // Ask the context to store it
+ int64_t id = 0;
+ try
+ {
+ id = rContext.AddFile(*dirstream, mDirectoryObjectID,
+ mModificationTime, mAttributesHash, mDiffFromFileID,
+ mFilename,
+ true /* mark files with same name as old versions */);
+ }
+ catch(BackupStoreException &e)
+ {
+ if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify)
+ {
+ return PROTOCOL_ERROR(Err_FileDoesNotVerify);
+ }
+ else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit)
+ {
+ return PROTOCOL_ERROR(Err_StorageLimitExceeded);
+ }
+ else
+ {
+ throw;
+ }
+ }
+
+ // Tell the caller what the file was
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Command to get an arbitary object from the server
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Check the object exists
+ if(!rContext.ObjectExists(mObjectID))
+ {
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(NoObject));
+ }
+
+ // Open the object
+ std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
+
+ // Stream it to the peer
+ rProtocol.SendStreamAfterCommand(object.release());
+
+ // Tell the caller what the file was
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Command to get an file object from the server -- may have to do a bit of
+// work to get the object.
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Check the objects exist
+ if(!rContext.ObjectExists(mObjectID)
+ || !rContext.ObjectExists(mInDirectory))
+ {
+ return PROTOCOL_ERROR(Err_DoesNotExist);
+ }
+
+ // Get the directory it's in
+ const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory));
+
+ // Find the object within the directory
+ BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID);
+ if(pfileEntry == 0)
+ {
+ return PROTOCOL_ERROR(Err_DoesNotExistInDirectory);
+ }
+
+ // The result
+ std::auto_ptr<IOStream> stream;
+
+ // Does this depend on anything?
+ if(pfileEntry->GetDependsNewer() != 0)
+ {
+ // File exists, but is a patch from a new version. Generate the older version.
+ std::vector<int64_t> patchChain;
+ int64_t id = mObjectID;
+ BackupStoreDirectory::Entry *en = 0;
+ do
+ {
+ patchChain.push_back(id);
+ en = rdir.FindEntryByID(id);
+ if(en == 0)
+ {
+ BOX_ERROR("Object " <<
+ BOX_FORMAT_OBJECTID(mObjectID) <<
+ " in dir " <<
+ BOX_FORMAT_OBJECTID(mInDirectory) <<
+ " for account " <<
+ BOX_FORMAT_ACCOUNT(rContext.GetClientID()) <<
+ " references object " <<
+ BOX_FORMAT_OBJECTID(id) <<
+ " which does not exist in dir");
+ return PROTOCOL_ERROR(Err_PatchConsistencyError);
+ }
+ id = en->GetDependsNewer();
+ }
+ while(en != 0 && id != 0);
+
+ // OK! The last entry in the chain is the full file, the others are patches back from it.
+ // Open the last one, which is the current from file
+ std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1]));
+
+ // Then, for each patch in the chain, do a combine
+ for(int p = ((int)patchChain.size()) - 2; p >= 0; --p)
+ {
+ // ID of patch
+ int64_t patchID = patchChain[p];
+
+ // Open it a couple of times
+ std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID));
+ std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID));
+
+ // Choose a temporary filename for the result of the combination
+ std::ostringstream fs;
+ fs << rContext.GetStoreRoot() << ".recombinetemp." << p;
+ std::string tempFn =
+ RaidFileController::DiscSetPathToFileSystemPath(
+ rContext.GetStoreDiscSet(), fs.str(),
+ p + 16);
+
+ // Open the temporary file
+ std::auto_ptr<IOStream> combined;
+ try
+ {
+ {
+ // Write nastily to allow this to work with gcc 2.x
+ std::auto_ptr<IOStream> t(
+ new InvisibleTempFileStream(
+ tempFn.c_str(),
+ O_RDWR | O_CREAT |
+ O_EXCL | O_BINARY |
+ O_TRUNC));
+ combined = t;
+ }
+ }
+ catch(...)
+ {
+ // Make sure it goes
+ ::unlink(tempFn.c_str());
+ throw;
+ }
+
+ // Do the combining
+ BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined);
+
+ // Move to the beginning of the combined file
+ combined->Seek(0, IOStream::SeekType_Absolute);
+
+ // Then shuffle round for the next go
+ if (from.get()) from->Close();
+ from = combined;
+ }
+
+ // Now, from contains a nice file to send to the client. Reorder it
+ {
+ // Write nastily to allow this to work with gcc 2.x
+ std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */));
+ stream = t;
+ }
+
+ // Release from file to avoid double deletion
+ from.release();
+ }
+ else
+ {
+ // Simple case: file already exists on disc ready to go
+
+ // Open the object
+ std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
+ BufferedStream buf(*object);
+
+ // Verify it
+ if(!BackupStoreFile::VerifyEncodedFileFormat(buf))
+ {
+ return PROTOCOL_ERROR(Err_FileDoesNotVerify);
+ }
+
+ // Reset stream -- seek to beginning
+ object->Seek(0, IOStream::SeekType_Absolute);
+
+ // Reorder the stream/file into stream order
+ {
+ // Write nastily to allow this to work with gcc 2.x
+ std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */));
+ stream = t;
+ }
+
+ // Object will be deleted when the stream is deleted,
+ // so can release the object auto_ptr here to avoid
+ // premature deletion
+ object.release();
+ }
+
+ // Stream the reordered stream to the peer
+ rProtocol.SendStreamAfterCommand(stream.get());
+
+ // Don't delete the stream here
+ stream.release();
+
+ // Tell the caller what the file was
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Create directory command
+// Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Get the stream containing the attributes
+ std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+ // Collect the attributes -- do this now so no matter what the outcome,
+ // the data has been absorbed.
+ StreamableMemBlock attr;
+ attr.Set(*attrstream, rProtocol.GetTimeout());
+
+ // Check to see if the hard limit has been exceeded
+ if(rContext.HardLimitExceeded())
+ {
+ // Won't allow creation if the limit has been exceeded
+ return PROTOCOL_ERROR(Err_StorageLimitExceeded);
+ }
+
+ bool alreadyExists = false;
+ int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists);
+
+ if(alreadyExists)
+ {
+ return PROTOCOL_ERROR(Err_DirectoryAlreadyExists);
+ }
+
+ // Tell the caller what the file was
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Change attributes on directory
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Get the stream containing the attributes
+ std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+ // Collect the attributes -- do this now so no matter what the outcome,
+ // the data has been absorbed.
+ StreamableMemBlock attr;
+ attr.Set(*attrstream, rProtocol.GetTimeout());
+
+ // Get the context to do it's magic
+ rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime);
+
+ // Tell the caller what the file was
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupStoreContext &)
+// Purpose: Change attributes on directory
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Get the stream containing the attributes
+ std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+ // Collect the attributes -- do this now so no matter what the outcome,
+ // the data has been absorbed.
+ StreamableMemBlock attr;
+ attr.Set(*attrstream, rProtocol.GetTimeout());
+
+ // Get the context to do it's magic
+ int64_t objectID = 0;
+ if(!rContext.ChangeFileAttributes(mFilename, mInDirectory, attr, mAttributesHash, objectID))
+ {
+ // Didn't exist
+ return PROTOCOL_ERROR(Err_DoesNotExist);
+ }
+
+ // Tell the caller what the file was
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Delete a file
+// Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Context handles this
+ int64_t objectID = 0;
+ rContext.DeleteFile(mFilename, mInDirectory, objectID);
+
+ // return the object ID or zero for not found
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerUndeleteFile::DoCommand(
+// BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Undelete a file
+// Created: 2008-09-12
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteFile::DoCommand(
+ BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Context handles this
+ bool result = rContext.UndeleteFile(mObjectID, mInDirectory);
+
+ // return the object ID or zero for not found
+ return std::auto_ptr<ProtocolObject>(
+ new BackupProtocolServerSuccess(result ? mObjectID : 0));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Delete a directory
+// Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Check it's not asking for the root directory to be deleted
+ if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+ {
+ return PROTOCOL_ERROR(Err_CannotDeleteRoot);
+ }
+
+ // Context handles this
+ try
+ {
+ rContext.DeleteDirectory(mObjectID);
+ }
+ catch (BackupStoreException &e)
+ {
+ if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject)
+ {
+ return PROTOCOL_ERROR(Err_MultiplyReferencedObject);
+ }
+
+ throw;
+ }
+
+ // return the object ID
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Undelete a directory
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Check it's not asking for the root directory to be deleted
+ if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+ {
+ return PROTOCOL_ERROR(Err_CannotDeleteRoot);
+ }
+
+ // Context handles this
+ rContext.DeleteDirectory(mObjectID, true /* undelete */);
+
+ // return the object ID
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Command to set the client's store marker
+// Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Set the marker
+ rContext.SetClientStoreMarker(mClientStoreMarker);
+
+ // return store marker set
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mClientStoreMarker));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Command to move an object from one directory to another
+// Created: 2003/11/12
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+ CHECK_WRITEABLE_SESSION
+
+ // Let context do this, but modify error reporting on exceptions...
+ try
+ {
+ rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory,
+ mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName,
+ (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject);
+ }
+ catch(BackupStoreException &e)
+ {
+ if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory)
+ {
+ return PROTOCOL_ERROR(Err_DoesNotExist);
+ }
+ else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory)
+ {
+ return PROTOCOL_ERROR(Err_TargetNameExists);
+ }
+ else
+ {
+ throw;
+ }
+ }
+
+ // Return the object ID
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Command to find the name of an object
+// Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Create a stream for the list of filenames
+ std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
+
+ // Object and directory IDs
+ int64_t objectID = mObjectID;
+ int64_t dirID = mContainingDirectoryID;
+
+ // Data to return in the reply
+ int32_t numNameElements = 0;
+ int16_t objectFlags = 0;
+ int64_t modTime = 0;
+ uint64_t attrModHash = 0;
+ bool haveModTimes = false;
+
+ do
+ {
+ // Check the directory really exists
+ if(!rContext.ObjectExists(dirID, BackupStoreContext::ObjectExists_Directory))
+ {
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
+ }
+
+ // Load up the directory
+ const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID));
+
+ // Find the element in this directory and store it's name
+ if(objectID != ObjectID_DirectoryOnly)
+ {
+ const BackupStoreDirectory::Entry *en = rdir.FindEntryByID(objectID);
+
+ // If this can't be found, then there is a problem... tell the caller it can't be found
+ if(en == 0)
+ {
+ // Abort!
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
+ }
+
+ // Store flags?
+ if(objectFlags == 0)
+ {
+ objectFlags = en->GetFlags();
+ }
+
+ // Store modification times?
+ if(!haveModTimes)
+ {
+ modTime = en->GetModificationTime();
+ attrModHash = en->GetAttributesHash();
+ haveModTimes = true;
+ }
+
+ // Store the name in the stream
+ en->GetName().WriteToStream(*stream);
+
+ // Count of name elements
+ ++numNameElements;
+ }
+
+ // Setup for next time round
+ objectID = dirID;
+ dirID = rdir.GetContainerID();
+
+ } while(objectID != 0 && objectID != BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+ // Stream to send?
+ if(numNameElements > 0)
+ {
+ // Get the stream ready to go
+ stream->SetForReading();
+ // Tell the protocol to send the stream
+ rProtocol.SendStreamAfterCommand(stream.release());
+ }
+
+ // Make reply
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(numNameElements, modTime, attrModHash, objectFlags));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Get the block index from a file, by ID
+// Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Open the file
+ std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID));
+
+ // Move the file pointer to the block index
+ BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
+
+ // Return the stream to the client
+ rProtocol.SendStreamAfterCommand(stream.release());
+
+ // Return the object ID
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Get the block index from a file, by name within a directory
+// Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Get the directory
+ const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory));
+
+ // Find the latest object ID within it which has the same name
+ int64_t objectID = 0;
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+ {
+ if(en->GetName() == mFilename)
+ {
+ // Store the ID, if it's a newer ID than the last one
+ if(en->GetObjectID() > objectID)
+ {
+ objectID = en->GetObjectID();
+ }
+ }
+ }
+
+ // Found anything?
+ if(objectID == 0)
+ {
+ // No... return a zero object ID
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(0));
+ }
+
+ // Open the file
+ std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID));
+
+ // Move the file pointer to the block index
+ BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
+
+ // Return the stream to the client
+ rProtocol.SendStreamAfterCommand(stream.release());
+
+ // Return the object ID
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Return the amount of disc space used
+// Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ // Get store info from context
+ const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo());
+
+ // Find block size
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber()));
+
+ // Return info
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerAccountUsage(
+ rinfo.GetBlocksUsed(),
+ rinfo.GetBlocksInOldFiles(),
+ rinfo.GetBlocksInDeletedFiles(),
+ rinfo.GetBlocksInDirectories(),
+ rinfo.GetBlocksSoftLimit(),
+ rinfo.GetBlocksHardLimit(),
+ rdiscSet.GetBlockSize()
+ ));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupStoreContext &)
+// Purpose: Return the amount of disc space used
+// Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext)
+{
+ CHECK_PHASE(Phase_Commands)
+
+ //
+ // NOOP
+ //
+ return std::auto_ptr<ProtocolObject>(new BackupProtocolServerIsAlive());
+}
Copied: box/trunk/lib/backupstore/BackupConstants.h (from rev 2943, box/trunk/bin/bbstored/BackupConstants.h)
===================================================================
--- box/trunk/lib/backupstore/BackupConstants.h (rev 0)
+++ box/trunk/lib/backupstore/BackupConstants.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,21 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupConstants.h
+// Purpose: Constants for the backup server and client
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCONSTANTS__H
+#define BACKUPCONSTANTS__H
+
+// 15 minutes to timeout (milliseconds)
+#define BACKUP_STORE_TIMEOUT (15*60*1000)
+
+// Should the store daemon convert files to Raid immediately?
+#define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true
+
+#endif // BACKUPCONSTANTS__H
+
+
Copied: box/trunk/lib/backupstore/BackupStoreConstants.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreConstants.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreConstants.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreConstants.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,44 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreContants.h
+// Purpose: constants for the backup system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONSTANTS__H
+#define BACKUPSTORECONSTANTS__H
+
+#define BACKUPSTORE_ROOT_DIRECTORY_ID 1
+
+#define BACKUP_STORE_SERVER_VERSION 1
+
+// Minimum size for a chunk to be compressed
+#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256
+
+// min and max sizes for blocks
+#define BACKUP_FILE_MIN_BLOCK_SIZE 4096
+#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024)
+
+// Increase the block size if there are more than this number of blocks
+#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096
+
+// Avoid creating blocks smaller than this
+#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128
+
+// Maximum number of sizes to do an rsync-like scan for
+#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64
+
+// When doing rsync scans, do not scan for blocks smaller than
+#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128
+
+// A limit to stop diffing running out of control: If more than this
+// times the number of blocks in the original index are found, stop
+// looking. This stops really bad cases of diffing files containing
+// all the same byte using huge amounts of memory and processor time.
+// This is a multiple of the number of blocks in the diff from file.
+#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096
+
+#endif // BACKUPSTORECONSTANTS__H
+
Copied: box/trunk/lib/backupstore/BackupStoreContext.cpp (from rev 2943, box/trunk/bin/bbstored/BackupStoreContext.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreContext.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreContext.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,1808 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreContext.cpp
+// Purpose: Context for backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BackupConstants.h"
+#include "BackupStoreContext.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreObjectMagic.h"
+#include "BufferedStream.h"
+#include "BufferedWriteStream.h"
+#include "FileStream.h"
+#include "InvisibleTempFileStream.h"
+#include "RaidFileController.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "StoreStructure.h"
+
+class BackupStoreDaemon;
+
+#include "MemLeakFindOn.h"
+
+// Maximum number of directories to keep in the cache
+// When the cache is bigger than this, everything gets
+// deleted.
+#ifdef BOX_RELEASE_BUILD
+ #define MAX_CACHE_SIZE 32
+#else
+ #define MAX_CACHE_SIZE 2
+#endif
+
+// Allow the housekeeping process 4 seconds to release an account
+#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4
+
+// Maximum amount of store info updates before it's actually saved to disc.
+#define STORE_INFO_SAVE_DELAY 96
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::BackupStoreContext()
+// Purpose: Constructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreContext::BackupStoreContext(int32_t ClientID,
+ HousekeepingInterface &rDaemon)
+ : mClientID(ClientID),
+ mrDaemon(rDaemon),
+ mProtocolPhase(Phase_START),
+ mClientHasAccount(false),
+ mStoreDiscSet(-1),
+ mReadOnly(true),
+ mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY),
+ mpTestHook(NULL)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::~BackupStoreContext()
+// Purpose: Destructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreContext::~BackupStoreContext()
+{
+ // Delete the objects in the cache
+ for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
+ {
+ delete (i->second);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::CleanUp()
+// Purpose: Clean up after a connection
+// Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::CleanUp()
+{
+ // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified
+ if(mapStoreInfo.get() && !(mapStoreInfo->IsReadOnly()) &&
+ mapStoreInfo->IsModified())
+ {
+ mapStoreInfo->Save();
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::ReceivedFinishCommand()
+// Purpose: Called when the finish command is received by the protocol
+// Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::ReceivedFinishCommand()
+{
+ if(!mReadOnly && mapStoreInfo.get())
+ {
+ // Save the store info, not delayed
+ SaveStoreInfo(false);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::AttemptToGetWriteLock()
+// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags
+// Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+bool BackupStoreContext::AttemptToGetWriteLock()
+{
+ // Make the filename of the write lock file
+ std::string writeLockFile;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile);
+
+ // Request the lock
+ bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
+
+ if(!gotLock)
+ {
+ // The housekeeping process might have the thing open -- ask it to stop
+ char msg[256];
+ int msgLen = sprintf(msg, "r%x\n", mClientID);
+ // Send message
+ mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen);
+
+ // Then try again a few times
+ int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT;
+ do
+ {
+ ::sleep(1 /* second */);
+ --tries;
+ gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
+
+ } while(!gotLock && tries > 0);
+ }
+
+ if(gotLock)
+ {
+ // Got the lock, mark as not read only
+ mReadOnly = false;
+ }
+
+ return gotLock;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::LoadStoreInfo()
+// Purpose: Load the store info from disc
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::LoadStoreInfo()
+{
+ if(mapStoreInfo.get() != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded)
+ }
+
+ // Load it up!
+ std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly));
+
+ // Check it
+ if(i->GetAccountID() != mClientID)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount)
+ }
+
+ // Keep the pointer to it
+ mapStoreInfo = i;
+
+ BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+ // try to load the reference count database
+ try
+ {
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database is missing or corrupted, "
+ "creating a new one, expect housekeeping to find and "
+ "fix problems with reference counts later.");
+
+ BackupStoreRefCountDatabase::CreateForRegeneration(account);
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::SaveStoreInfo(bool)
+// Purpose: Potentially delayed saving of the store info
+// Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::SaveStoreInfo(bool AllowDelay)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // Can delay saving it a little while?
+ if(AllowDelay)
+ {
+ --mSaveStoreInfoDelay;
+ if(mSaveStoreInfoDelay > 0)
+ {
+ return;
+ }
+ }
+
+ // Want to save now
+ mapStoreInfo->Save();
+
+ // Set count for next delay
+ mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool)
+// Purpose: Create the filename of an object in the store, optionally creating the
+// containing directory if it doesn't already exist.
+// Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists)
+{
+ // Delegate to utility function
+ StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::GetDirectoryInternal(int64_t)
+// Purpose: Return a reference to a directory. Valid only until the
+// next time a function which affects directories is called.
+// Mainly this funciton, and creation of files.
+// Private version of this, which returns non-const directories.
+// Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID)
+{
+ // Get the filename
+ std::string filename;
+ MakeObjectFilename(ObjectID, filename);
+
+ // Already in cache?
+ std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
+ if(item != mDirectoryCache.end())
+ {
+ // Check the revision ID of the file -- does it need refreshing?
+ int64_t revID = 0;
+ if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID))
+ {
+ THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted)
+ }
+
+ if(revID == item->second->GetRevisionID())
+ {
+ // Looks good... return the cached object
+ BOX_TRACE("Returning object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " from cache, modtime = " << revID);
+ return *(item->second);
+ }
+
+ BOX_TRACE("Refreshing object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " in cache, modtime changed from " <<
+ item->second->GetRevisionID() << " to " << revID);
+
+ // Delete this cached object
+ delete item->second;
+ mDirectoryCache.erase(item);
+ }
+
+ // Need to load it up
+
+ // First check to see if the cache is too big
+ if(mDirectoryCache.size() > MAX_CACHE_SIZE)
+ {
+ // Very simple. Just delete everything!
+ for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
+ {
+ delete (i->second);
+ }
+ mDirectoryCache.clear();
+ }
+
+ // Get a RaidFileRead to read it
+ int64_t revID = 0;
+ std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID));
+ ASSERT(revID != 0);
+
+ // New directory object
+ std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory);
+
+ // Read it from the stream, then set it's revision ID
+ BufferedStream buf(*objectFile);
+ dir->ReadFromStream(buf, IOStream::TimeOutInfinite);
+ dir->SetRevisionID(revID);
+
+ // Make sure the size of the directory is available for writing the dir back
+ int64_t dirSize = objectFile->GetDiscUsageInBlocks();
+ ASSERT(dirSize > 0);
+ dir->SetUserInfo1_SizeInBlocks(dirSize);
+
+ // Store in cache
+ BackupStoreDirectory *pdir = dir.release();
+ try
+ {
+ mDirectoryCache[ObjectID] = pdir;
+ }
+ catch(...)
+ {
+ delete pdir;
+ throw;
+ }
+
+ // Return it
+ return *pdir;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::AllocateObjectID()
+// Purpose: Allocate a new object ID, tolerant of failures to save store info
+// Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreContext::AllocateObjectID()
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY
+ // times after it has been updated, this is a reasonable number of times
+ // to try for finding an unused ID.
+ // (Sizes used in the store info are fixed by the housekeeping process)
+ int retryLimit = (STORE_INFO_SAVE_DELAY * 2);
+
+ while(retryLimit > 0)
+ {
+ // Attempt to allocate an ID from the store
+ int64_t id = mapStoreInfo->AllocateObjectID();
+
+ // Generate filename
+ std::string filename;
+ MakeObjectFilename(id, filename);
+ // Check it doesn't exist
+ if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
+ {
+ // Success!
+ return id;
+ }
+
+ // Decrement retry count, and try again
+ --retryLimit;
+
+ // Mark that the store info should be saved as soon as possible
+ mSaveStoreInfoDelay = 0;
+
+ BOX_WARNING("When allocating object ID, found that " <<
+ BOX_FORMAT_OBJECTID(id) << " is already in use");
+ }
+
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::AddFile(IOStream &, int64_t,
+// int64_t, int64_t, const BackupStoreFilename &, bool)
+// Purpose: Add a file to the store, from a given stream, into
+// a specified directory. Returns object ID of the new
+// file.
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+ int64_t ModificationTime, int64_t AttributesHash,
+ int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+ bool MarkFileWithSameNameAsOldVersions)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // This is going to be a bit complex to make sure it copes OK
+ // with things going wrong.
+ // The only thing which isn't safe is incrementing the object ID
+ // and keeping the blocks used entirely accurate -- but these
+ // aren't big problems if they go horribly wrong. The sizes will
+ // be corrected the next time the account has a housekeeping run,
+ // and the object ID allocation code is tolerant of missed IDs.
+ // (the info is written lazily, so these are necessary)
+
+ // Get the directory we want to modify
+ BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+ // Allocate the next ID
+ int64_t id = AllocateObjectID();
+
+ // Stream the file to disc
+ std::string fn;
+ MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
+ int64_t newObjectBlocksUsed = 0;
+ RaidFileWrite *ppreviousVerStoreFile = 0;
+ bool reversedDiffIsCompletelyDifferent = false;
+ int64_t oldVersionNewBlocksUsed = 0;
+ try
+ {
+ RaidFileWrite storeFile(mStoreDiscSet, fn);
+ storeFile.Open(false /* no overwriting */);
+
+ // size adjustment from use of patch in old file
+ int64_t spaceSavedByConversionToPatch = 0;
+
+ // Diff or full file?
+ if(DiffFromFileID == 0)
+ {
+ // A full file, just store to disc
+ if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT))
+ {
+ THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
+ }
+ }
+ else
+ {
+ // Check that the diffed from ID actually exists in the directory
+ if(dir.FindEntryByID(DiffFromFileID) == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory)
+ }
+
+ // Diff file, needs to be recreated.
+ // Choose a temporary filename.
+ std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp",
+ 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */));
+
+ try
+ {
+ // Open it twice
+#ifdef WIN32
+ InvisibleTempFileStream diff(tempFn.c_str(),
+ O_RDWR | O_CREAT | O_BINARY);
+ InvisibleTempFileStream diff2(tempFn.c_str(),
+ O_RDWR | O_BINARY);
+#else
+ FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL);
+ FileStream diff2(tempFn.c_str(), O_RDONLY);
+
+ // Unlink it immediately, so it definitely goes away
+ if(::unlink(tempFn.c_str()) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+#endif
+
+ // Stream the incoming diff to this temporary file
+ if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT))
+ {
+ THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
+ }
+
+ // Verify the diff
+ diff.Seek(0, IOStream::SeekType_Absolute);
+ if(!BackupStoreFile::VerifyEncodedFileFormat(diff))
+ {
+ THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
+ }
+
+ // Seek to beginning of diff file
+ diff.Seek(0, IOStream::SeekType_Absolute);
+
+ // Filename of the old version
+ std::string oldVersionFilename;
+ MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */);
+
+ // Reassemble that diff -- open previous file, and combine the patch and file
+ std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
+ BackupStoreFile::CombineFile(diff, diff2, *from, storeFile);
+
+ // Then... reverse the patch back (open the from file again, and create a write file to overwrite it)
+ std::auto_ptr<RaidFileRead> from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
+ ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename);
+ ppreviousVerStoreFile->Open(true /* allow overwriting */);
+ from->Seek(0, IOStream::SeekType_Absolute);
+ diff.Seek(0, IOStream::SeekType_Absolute);
+ BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile,
+ DiffFromFileID, &reversedDiffIsCompletelyDifferent);
+
+ // Store disc space used
+ oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks();
+
+ // And make a space adjustment for the size calculation
+ spaceSavedByConversionToPatch =
+ from->GetDiscUsageInBlocks() -
+ oldVersionNewBlocksUsed;
+
+ // Everything cleans up here...
+ }
+ catch(...)
+ {
+ // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway
+ ::unlink(tempFn.c_str());
+ throw;
+ }
+ }
+
+ // Get the blocks used
+ newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks();
+
+ // Exceeds the hard limit?
+ int64_t newBlocksUsed = mapStoreInfo->GetBlocksUsed() +
+ newObjectBlocksUsed - spaceSavedByConversionToPatch;
+ if(newBlocksUsed > mapStoreInfo->GetBlocksHardLimit())
+ {
+ THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit)
+ // The store file will be deleted automatically by the RaidFile object
+ }
+
+ // Commit the file
+ storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+ }
+ catch(...)
+ {
+ // Delete any previous version store file
+ if(ppreviousVerStoreFile != 0)
+ {
+ delete ppreviousVerStoreFile;
+ ppreviousVerStoreFile = 0;
+ }
+
+ throw;
+ }
+
+ // Verify the file -- only necessary for non-diffed versions
+ // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because
+ // in the non-diffed code path it's never allocated.
+ if(DiffFromFileID == 0)
+ {
+ std::auto_ptr<RaidFileRead> checkFile(RaidFileRead::Open(mStoreDiscSet, fn));
+ if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile))
+ {
+ // Error! Delete the file
+ RaidFileWrite del(mStoreDiscSet, fn);
+ del.Delete();
+
+ // Exception
+ THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
+ }
+ }
+
+ // Modify the directory -- first make all files with the same name
+ // marked as an old version
+ int64_t blocksInOldFiles = 0;
+ try
+ {
+ if(MarkFileWithSameNameAsOldVersions)
+ {
+ BackupStoreDirectory::Iterator i(dir);
+
+ BackupStoreDirectory::Entry *e = 0;
+ while((e = i.Next()) != 0)
+ {
+ // First, check it's not an old version (cheaper comparison)
+ if(! e->IsOld())
+ {
+ // Compare name
+ if(e->GetName() == rFilename)
+ {
+ // Check that it's definately not an old version
+ ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0);
+ // Set old version flag
+ e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion);
+ // Can safely do this, because we know we won't be here if it's already
+ // an old version
+ blocksInOldFiles += e->GetSizeInBlocks();
+ }
+ }
+ }
+ }
+
+ // Then the new entry
+ BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename,
+ ModificationTime, id, newObjectBlocksUsed,
+ BackupStoreDirectory::Entry::Flags_File,
+ AttributesHash);
+
+ // Adjust for the patch back stuff?
+ if(DiffFromFileID != 0)
+ {
+ // Get old version entry
+ BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID);
+ ASSERT(poldEntry != 0);
+
+ // Adjust dependency info of file?
+ if(!reversedDiffIsCompletelyDifferent)
+ {
+ poldEntry->SetDependsNewer(id);
+ pnewEntry->SetDependsOlder(DiffFromFileID);
+ }
+
+ // Adjust size of old entry
+ int64_t oldSize = poldEntry->GetSizeInBlocks();
+ poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed);
+
+ // And adjust blocks used count, for later adjustment
+ newObjectBlocksUsed += (oldVersionNewBlocksUsed - oldSize);
+ blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize);
+ }
+
+ // Write the directory back to disc
+ SaveDirectory(dir, InDirectory);
+
+ // Commit the old version's new patched version, now that the directory safely reflects
+ // the state of the files on disc.
+ if(ppreviousVerStoreFile != 0)
+ {
+ ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+ delete ppreviousVerStoreFile;
+ ppreviousVerStoreFile = 0;
+ }
+ }
+ catch(...)
+ {
+ // Back out on adding that file
+ RaidFileWrite del(mStoreDiscSet, fn);
+ del.Delete();
+
+ // Remove this entry from the cache
+ RemoveDirectoryFromCache(InDirectory);
+
+ // Delete any previous version store file
+ if(ppreviousVerStoreFile != 0)
+ {
+ delete ppreviousVerStoreFile;
+ ppreviousVerStoreFile = 0;
+ }
+
+ // Don't worry about the incremented number in the store info
+ throw;
+ }
+
+ // Check logic
+ ASSERT(ppreviousVerStoreFile == 0);
+
+ // Modify the store info
+
+ if(DiffFromFileID == 0)
+ {
+ mapStoreInfo->AdjustNumFiles(1);
+ }
+ else
+ {
+ mapStoreInfo->AdjustNumOldFiles(1);
+ }
+
+ mapStoreInfo->ChangeBlocksUsed(newObjectBlocksUsed);
+ mapStoreInfo->ChangeBlocksInCurrentFiles(newObjectBlocksUsed -
+ blocksInOldFiles);
+ mapStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
+
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
+
+ // Save the store info -- can cope if this exceptions because infomation
+ // will be rebuilt by housekeeping, and ID allocation can recover.
+ SaveStoreInfo(false);
+
+ // Return the ID to the caller
+ return id;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
+// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+// Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut)
+{
+ // Essential checks!
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // Find the directory the file is in (will exception if it fails)
+ BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+ // Setup flags
+ bool fileExisted = false;
+ bool madeChanges = false;
+ rObjectIDOut = 0; // not found
+
+ // Count of deleted blocks
+ int64_t blocksDel = 0;
+
+ try
+ {
+ // Iterate through directory, only looking at files which haven't been deleted
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *e = 0;
+ while((e = i.Next(BackupStoreDirectory::Entry::Flags_File,
+ BackupStoreDirectory::Entry::Flags_Deleted)) != 0)
+ {
+ // Compare name
+ if(e->GetName() == rFilename)
+ {
+ // Check that it's definately not already deleted
+ ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0);
+ // Set deleted flag
+ e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+ // Mark as made a change
+ madeChanges = true;
+ // Can safely do this, because we know we won't be here if it's already
+ // an old version
+ blocksDel += e->GetSizeInBlocks();
+ // Is this the last version?
+ if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
+ {
+ // Yes. It's been found.
+ rObjectIDOut = e->GetObjectID();
+ fileExisted = true;
+ }
+ }
+ }
+
+ // Save changes?
+ if(madeChanges)
+ {
+ // Save the directory back
+ SaveDirectory(dir, InDirectory);
+
+ // Modify the store info, and write
+ // It definitely wasn't an old or deleted version
+ mapStoreInfo->AdjustNumFiles(-1);
+ mapStoreInfo->AdjustNumDeletedFiles(1);
+ mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel);
+
+ SaveStoreInfo(false);
+ }
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(InDirectory);
+ throw;
+ }
+
+ return fileExisted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t)
+// Purpose: Undeletes a file, if it exists, returning true if
+// the file existed.
+// Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory)
+{
+ // Essential checks!
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // Find the directory the file is in (will exception if it fails)
+ BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+ // Setup flags
+ bool fileExisted = false;
+ bool madeChanges = false;
+
+ // Count of deleted blocks
+ int64_t blocksDel = 0;
+
+ try
+ {
+ // Iterate through directory, only looking at files which have been deleted
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *e = 0;
+ while((e = i.Next(BackupStoreDirectory::Entry::Flags_File |
+ BackupStoreDirectory::Entry::Flags_Deleted, 0)) != 0)
+ {
+ // Compare name
+ if(e->GetObjectID() == ObjectID)
+ {
+ // Check that it's definitely already deleted
+ ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
+ // Clear deleted flag
+ e->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+ // Mark as made a change
+ madeChanges = true;
+ blocksDel -= e->GetSizeInBlocks();
+
+ // Is this the last version?
+ if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
+ {
+ // Yes. It's been found.
+ fileExisted = true;
+ }
+ }
+ }
+
+ // Save changes?
+ if(madeChanges)
+ {
+ // Save the directory back
+ SaveDirectory(dir, InDirectory);
+
+ // Modify the store info, and write
+ mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel);
+
+ // Maybe postponed save of store info
+ SaveStoreInfo();
+ }
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(InDirectory);
+ throw;
+ }
+
+ return fileExisted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::RemoveDirectoryFromCache(int64_t)
+// Purpose: Remove directory from cache
+// Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID)
+{
+ std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
+ if(item != mDirectoryCache.end())
+ {
+ // Delete this cached object
+ delete item->second;
+ // Erase the entry form the map
+ mDirectoryCache.erase(item);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t)
+// Purpose: Save directory back to disc, update time in cache
+// Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(rDir.GetObjectID() != ObjectID)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ try
+ {
+ // Write to disc, adjust size in store info
+ std::string dirfn;
+ MakeObjectFilename(ObjectID, dirfn);
+ {
+ RaidFileWrite writeDir(mStoreDiscSet, dirfn);
+ writeDir.Open(true /* allow overwriting */);
+
+ BufferedWriteStream buffer(writeDir);
+ rDir.WriteToStream(buffer);
+ buffer.Flush();
+
+ // get the disc usage (must do this before commiting it)
+ int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+ // Commit directory
+ writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+ // Make sure the size of the directory is available for writing the dir back
+ ASSERT(dirSize > 0);
+ int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks();
+ mapStoreInfo->ChangeBlocksUsed(sizeAdjustment);
+ mapStoreInfo->ChangeBlocksInDirectories(sizeAdjustment);
+ // Update size stored in directory
+ rDir.SetUserInfo1_SizeInBlocks(dirSize);
+ }
+ // Refresh revision ID in cache
+ {
+ int64_t revid = 0;
+ if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ rDir.SetRevisionID(revid);
+ }
+ }
+ catch(...)
+ {
+ // Remove it from the cache if anything went wrong
+ RemoveDirectoryFromCache(ObjectID);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::AddDirectory(int64_t,
+// const BackupStoreFilename &, bool &)
+// Purpose: Creates a directory (or just returns the ID of an
+// existing one). rAlreadyExists set appropraitely.
+// Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // Flags as not already existing
+ rAlreadyExists = false;
+
+ // Get the directory we want to modify
+ BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+ // Scan the directory for the name (only looking for directories which already exist)
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING,
+ BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories
+ {
+ if(en->GetName() == rFilename)
+ {
+ // Already exists
+ rAlreadyExists = true;
+ return en->GetObjectID();
+ }
+ }
+ }
+
+ // Allocate the next ID
+ int64_t id = AllocateObjectID();
+
+ // Create an empty directory with the given attributes on disc
+ std::string fn;
+ MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
+ {
+ BackupStoreDirectory emptyDir(id, InDirectory);
+ // add the atttribues
+ emptyDir.SetAttributes(Attributes, AttributesModTime);
+
+ // Write...
+ RaidFileWrite dirFile(mStoreDiscSet, fn);
+ dirFile.Open(false /* no overwriting */);
+ emptyDir.WriteToStream(dirFile);
+ // Get disc usage, before it's commited
+ int64_t dirSize = dirFile.GetDiscUsageInBlocks();
+ // Commit the file
+ dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+ // Make sure the size of the directory is added to the usage counts in the info
+ ASSERT(dirSize > 0);
+ mapStoreInfo->ChangeBlocksUsed(dirSize);
+ mapStoreInfo->ChangeBlocksInDirectories(dirSize);
+ // Not added to cache, so don't set the size in the directory
+ }
+
+ // Then add it into the parent directory
+ try
+ {
+ dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
+ SaveDirectory(dir, InDirectory);
+
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
+ }
+ catch(...)
+ {
+ // Back out on adding that directory
+ RaidFileWrite del(mStoreDiscSet, fn);
+ del.Delete();
+
+ // Remove this entry from the cache
+ RemoveDirectoryFromCache(InDirectory);
+
+ // Don't worry about the incremented number in the store info
+ throw;
+ }
+
+ // Save the store info (may not be postponed)
+ mapStoreInfo->AdjustNumDirectories(1);
+ SaveStoreInfo(false);
+
+ // tell caller what the ID was
+ return id;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool)
+// Purpose: Recusively deletes a directory (or undeletes if Undelete = true)
+// Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete)
+{
+ // Essential checks!
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // Containing directory
+ int64_t InDirectory = 0;
+
+ // Count of blocks deleted
+ int64_t blocksDeleted = 0;
+
+ try
+ {
+ // Get the directory that's to be deleted
+ {
+ // In block, because dir may not be valid after the delete directory call
+ BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+
+ // Store the directory it's in for later
+ InDirectory = dir.GetContainerID();
+
+ // Depth first delete of contents
+ DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete);
+ }
+
+ // Remove the entry from the directory it's in
+ ASSERT(InDirectory != 0);
+ BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory));
+
+ BackupStoreDirectory::Iterator i(parentDir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
+ Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete)
+ {
+ if(en->GetObjectID() == ObjectID)
+ {
+ // This is the one to delete
+ if(Undelete)
+ {
+ en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+ }
+ else
+ {
+ en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+ }
+
+ // Save it
+ SaveDirectory(parentDir, InDirectory);
+
+ // Done
+ break;
+ }
+ }
+
+ // Update blocks deleted count
+ mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted));
+ mapStoreInfo->AdjustNumDirectories(-1);
+ SaveStoreInfo(false);
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(InDirectory);
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t)
+// Purpose: Private. Deletes a directory depth-first recusively.
+// Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete)
+{
+ try
+ {
+ // Does things carefully to avoid using a directory in the cache after recursive call
+ // because it may have been deleted.
+
+ // Do sub directories
+ {
+ // Get the directory...
+ BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+
+ // Then scan it for directories
+ std::vector<int64_t> subDirs;
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ if(Undelete)
+ {
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs
+ BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0)
+ {
+ // Store the directory ID.
+ subDirs.push_back(en->GetObjectID());
+ }
+ }
+ else
+ {
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only
+ BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones
+ {
+ // Store the directory ID.
+ subDirs.push_back(en->GetObjectID());
+ }
+ }
+
+ // Done with the directory for now. Recurse to sub directories
+ for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i)
+ {
+ DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete);
+ }
+ }
+
+ // Then, delete the files. Will need to load the directory again because it might have
+ // been removed from the cache.
+ {
+ // Get the directory...
+ BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+
+ // Changes made?
+ bool changesMade = false;
+
+ // Run through files
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+
+ while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
+ Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete)
+ {
+ // Add/remove the deleted flags
+ if(Undelete)
+ {
+ en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+ }
+ else
+ {
+ en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+ }
+
+ // Keep count of the deleted blocks
+ if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
+ {
+ rBlocksDeletedOut += en->GetSizeInBlocks();
+ }
+
+ // Did something
+ changesMade = true;
+ }
+
+ // Save the directory
+ if(changesMade)
+ {
+ SaveDirectory(dir, ObjectID);
+ }
+ }
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(ObjectID);
+ throw;
+ }
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t)
+// Purpose: Change the attributes of a directory
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ try
+ {
+ // Get the directory we want to modify
+ BackupStoreDirectory &dir(GetDirectoryInternal(Directory));
+
+ // Set attributes
+ dir.SetAttributes(Attributes, AttributesModTime);
+
+ // Save back
+ SaveDirectory(dir, Directory);
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(Directory);
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t)
+// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't.
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ try
+ {
+ // Get the directory we want to modify
+ BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+ // Find the file entry
+ BackupStoreDirectory::Entry *en = 0;
+ // Iterate through current versions of files, only
+ BackupStoreDirectory::Iterator i(dir);
+ while((en = i.Next(
+ BackupStoreDirectory::Entry::Flags_File,
+ BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)
+ ) != 0)
+ {
+ if(en->GetName() == rFilename)
+ {
+ // Set attributes
+ en->SetAttributes(Attributes, AttributesHash);
+
+ // Tell caller the object ID
+ rObjectIDOut = en->GetObjectID();
+
+ // Done
+ break;
+ }
+ }
+ if(en == 0)
+ {
+ // Didn't find it
+ return false;
+ }
+
+ // Save back
+ SaveDirectory(dir, InDirectory);
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(InDirectory);
+ throw;
+ }
+
+ // Changed, everything OK
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::ObjectExists(int64_t)
+// Purpose: Test to see if an object of this ID exists in the store
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ // Note that we need to allow object IDs a little bit greater than the last one in the store info,
+ // because the store info may not have got saved in an error condition. Max greater ID is
+ // STORE_INFO_SAVE_DELAY in this case, *2 to be safe.
+ if(ObjectID <= 0 || ObjectID > (mapStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2)))
+ {
+ // Obviously bad object ID
+ return false;
+ }
+
+ // Test to see if it exists on the disc
+ std::string filename;
+ MakeObjectFilename(ObjectID, filename);
+ if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
+ {
+ // RaidFile reports no file there
+ return false;
+ }
+
+ // Do we need to be more specific?
+ if(MustBe != ObjectExists_Anything)
+ {
+ // Open the file
+ std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename));
+
+ // Read the first integer
+ u_int32_t magic;
+ if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */))
+ {
+ // Failed to get any bytes, must have failed
+ return false;
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0)
+ {
+ // Old version detected
+ return true;
+ }
+#endif
+
+ // Right one?
+ u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE;
+
+ // Check
+ if(ntohl(magic) != requiredMagic)
+ {
+ return false;
+ }
+
+ // File is implicitly closed
+ }
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::OpenObject(int64_t)
+// Purpose: Opens an object
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreContext::OpenObject(int64_t ObjectID)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ // Attempt to open the file
+ std::string fn;
+ MakeObjectFilename(ObjectID, fn);
+ return std::auto_ptr<IOStream>(RaidFileRead::Open(mStoreDiscSet, fn).release());
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::GetClientStoreMarker()
+// Purpose: Retrieve the client store marker
+// Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreContext::GetClientStoreMarker()
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ return mapStoreInfo->GetClientStoreMarker();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &)
+// Purpose: Get disc usage info from store info
+// Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ rBlocksUsed = mapStoreInfo->GetBlocksUsed();
+ rBlocksSoftLimit = mapStoreInfo->GetBlocksSoftLimit();
+ rBlocksHardLimit = mapStoreInfo->GetBlocksHardLimit();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::HardLimitExceeded()
+// Purpose: Returns true if the hard limit has been exceeded
+// Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreContext::HardLimitExceeded()
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ return mapStoreInfo->GetBlocksUsed() > mapStoreInfo->GetBlocksHardLimit();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::SetClientStoreMarker(int64_t)
+// Purpose: Sets the client store marker, and commits it to disc
+// Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ mapStoreInfo->SetClientStoreMarker(ClientStoreMarker);
+ SaveStoreInfo(false /* don't delay saving this */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
+// Purpose: Move an object (and all objects with the same name) from one directory to another
+// Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+ }
+
+ // Should deleted files be excluded when checking for the existance of objects with the target name?
+ int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject)
+ ?(BackupStoreDirectory::Entry::Flags_Deleted)
+ :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
+
+ // Special case if the directories are the same...
+ if(MoveFromDirectory == MoveToDirectory)
+ {
+ try
+ {
+ // Get the first directory
+ BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory));
+
+ // Find the file entry
+ BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID);
+
+ // Error if not found
+ if(en == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+ }
+
+ // Check the new name doens't already exist (optionally ignoring deleted files)
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *c = 0;
+ while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
+ {
+ if(c->GetName() == rNewFilename)
+ {
+ THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
+ }
+ }
+ }
+
+ // Need to get all the entries with the same name?
+ if(MoveAllWithSameName)
+ {
+ // Iterate through the directory, copying all with matching names
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *c = 0;
+ while((c = i.Next()) != 0)
+ {
+ if(c->GetName() == en->GetName())
+ {
+ // Rename this one
+ c->SetName(rNewFilename);
+ }
+ }
+ }
+ else
+ {
+ // Just copy this one
+ en->SetName(rNewFilename);
+ }
+
+ // Save the directory back
+ SaveDirectory(dir, MoveFromDirectory);
+ }
+ catch(...)
+ {
+ RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same
+ throw;
+ }
+
+ return;
+ }
+
+ // Got to be careful how this is written, as we can't guarentte that if we have two
+ // directories open, the first won't be deleted as the second is opened. (cache)
+
+ // List of entries to move
+ std::vector<BackupStoreDirectory::Entry *> moving;
+
+ // list of directory IDs which need to have containing dir id changed
+ std::vector<int64_t> dirsToChangeContainingID;
+
+ try
+ {
+ // First of all, get copies of the entries to move to the to directory.
+
+ {
+ // Get the first directory
+ BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
+
+ // Find the file entry
+ BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID);
+
+ // Error if not found
+ if(en == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+ }
+
+ // Need to get all the entries with the same name?
+ if(MoveAllWithSameName)
+ {
+ // Iterate through the directory, copying all with matching names
+ BackupStoreDirectory::Iterator i(from);
+ BackupStoreDirectory::Entry *c = 0;
+ while((c = i.Next()) != 0)
+ {
+ if(c->GetName() == en->GetName())
+ {
+ // Copy
+ moving.push_back(new BackupStoreDirectory::Entry(*c));
+
+ // Check for containing directory correction
+ if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID());
+ }
+ }
+ ASSERT(!moving.empty());
+ }
+ else
+ {
+ // Just copy this one
+ moving.push_back(new BackupStoreDirectory::Entry(*en));
+
+ // Check for containing directory correction
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID());
+ }
+ }
+
+ // Secondly, insert them into the to directory, and save it
+
+ {
+ // To directory
+ BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
+
+ // Check the new name doens't already exist
+ {
+ BackupStoreDirectory::Iterator i(to);
+ BackupStoreDirectory::Entry *c = 0;
+ while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
+ {
+ if(c->GetName() == rNewFilename)
+ {
+ THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
+ }
+ }
+ }
+
+ // Copy the entries into it, changing the name as we go
+ for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+ {
+ BackupStoreDirectory::Entry *en = (*i);
+ en->SetName(rNewFilename);
+ to.AddEntry(*en); // adds copy
+ }
+
+ // Save back
+ SaveDirectory(to, MoveToDirectory);
+ }
+
+ // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory
+ try
+ {
+ // Get directory
+ BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
+
+ // Delete each one
+ for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+ {
+ from.DeleteEntry((*i)->GetObjectID());
+ }
+
+ // Save back
+ SaveDirectory(from, MoveFromDirectory);
+ }
+ catch(...)
+ {
+ // UNDO modification to To directory
+
+ // Get directory
+ BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
+
+ // Delete each one
+ for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+ {
+ to.DeleteEntry((*i)->GetObjectID());
+ }
+
+ // Save back
+ SaveDirectory(to, MoveToDirectory);
+
+ // Throw the error
+ throw;
+ }
+
+ // Finally... for all the directories we moved, modify their containing directory ID
+ for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
+ {
+ // Load the directory
+ BackupStoreDirectory &change(GetDirectoryInternal(*i));
+
+ // Modify containing dir ID
+ change.SetContainerID(MoveToDirectory);
+
+ // Save it back
+ SaveDirectory(change, *i);
+ }
+ }
+ catch(...)
+ {
+ // Make sure directories aren't in the cache, as they may have been modified
+ RemoveDirectoryFromCache(MoveToDirectory);
+ RemoveDirectoryFromCache(MoveFromDirectory);
+ for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
+ {
+ RemoveDirectoryFromCache(*i);
+ }
+
+ while(!moving.empty())
+ {
+ delete moving.back();
+ moving.pop_back();
+ }
+ throw;
+ }
+
+ // Clean up
+ while(!moving.empty())
+ {
+ delete moving.back();
+ moving.pop_back();
+ }
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreContext::GetBackupStoreInfo()
+// Purpose: Return the backup store info object, exception if it isn't loaded
+// Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const
+{
+ if(mapStoreInfo.get() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+ }
+
+ return *(mapStoreInfo.get());
+}
+
+
Copied: box/trunk/lib/backupstore/BackupStoreContext.h (from rev 2943, box/trunk/bin/bbstored/BackupStoreContext.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreContext.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreContext.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,186 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreContext.h
+// Purpose: Context for backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCONTEXT__H
+#define BACKUPCONTEXT__H
+
+#include <string>
+#include <map>
+#include <memory>
+
+#include "BackupStoreRefCountDatabase.h"
+#include "NamedLock.h"
+#include "ProtocolObject.h"
+#include "Utils.h"
+
+class BackupStoreDirectory;
+class BackupStoreFilename;
+class BackupStoreDaemon;
+class BackupStoreInfo;
+class IOStream;
+class BackupProtocolObject;
+class StreamableMemBlock;
+
+class HousekeepingInterface
+{
+ public:
+ virtual ~HousekeepingInterface() { }
+ virtual void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreContext
+// Purpose: Context for backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreContext
+{
+public:
+ BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon);
+ ~BackupStoreContext();
+private:
+ BackupStoreContext(const BackupStoreContext &rToCopy);
+public:
+
+ void ReceivedFinishCommand();
+ void CleanUp();
+
+ int32_t GetClientID() {return mClientID;}
+
+ enum
+ {
+ Phase_START = 0,
+ Phase_Version = 0,
+ Phase_Login = 1,
+ Phase_Commands = 2
+ };
+
+ int GetPhase() const {return mProtocolPhase;}
+ void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;}
+
+ // Read only locking
+ bool SessionIsReadOnly() {return mReadOnly;}
+ bool AttemptToGetWriteLock();
+
+ void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;}
+ bool GetClientHasAccount() const {return mClientHasAccount;}
+ const std::string &GetStoreRoot() const {return mStoreRoot;}
+ int GetStoreDiscSet() const {return mStoreDiscSet;}
+
+ // Store info
+ void LoadStoreInfo();
+ void SaveStoreInfo(bool AllowDelay = true);
+ const BackupStoreInfo &GetBackupStoreInfo() const;
+
+ // Client marker
+ int64_t GetClientStoreMarker();
+ void SetClientStoreMarker(int64_t ClientStoreMarker);
+
+ // Usage information
+ void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit);
+ bool HardLimitExceeded();
+
+ // Reading directories
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BackupStoreContext::GetDirectory(int64_t)
+ // Purpose: Return a reference to a directory. Valid only until the
+ // next time a function which affects directories is called.
+ // Mainly this funciton, and creation of files.
+ // Created: 2003/09/02
+ //
+ // --------------------------------------------------------------------------
+ const BackupStoreDirectory &GetDirectory(int64_t ObjectID)
+ {
+ // External callers aren't allowed to change it -- this function
+ // merely turns the the returned directory const.
+ return GetDirectoryInternal(ObjectID);
+ }
+
+ // Manipulating files/directories
+ int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions);
+ int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists);
+ void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime);
+ bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut);
+ bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut);
+ bool UndeleteFile(int64_t ObjectID, int64_t InDirectory);
+ void DeleteDirectory(int64_t ObjectID, bool Undelete = false);
+ void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject);
+
+ // Manipulating objects
+ enum
+ {
+ ObjectExists_Anything = 0,
+ ObjectExists_File = 1,
+ ObjectExists_Directory = 2
+ };
+ bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything);
+ std::auto_ptr<IOStream> OpenObject(int64_t ObjectID);
+
+ // Info
+ int32_t GetClientID() const {return mClientID;}
+
+private:
+ void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false);
+ BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID);
+ void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID);
+ void RemoveDirectoryFromCache(int64_t ObjectID);
+ void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
+ int64_t AllocateObjectID();
+
+ int32_t mClientID;
+ HousekeepingInterface &mrDaemon;
+ int mProtocolPhase;
+ bool mClientHasAccount;
+ std::string mStoreRoot; // has final directory separator
+ int mStoreDiscSet;
+ bool mReadOnly;
+ NamedLock mWriteLock;
+ int mSaveStoreInfoDelay; // how many times to delay saving the store info
+
+ // Store info
+ std::auto_ptr<BackupStoreInfo> mapStoreInfo;
+
+ // Refcount database
+ std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
+
+ // Directory cache
+ std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
+
+public:
+ class TestHook
+ {
+ public:
+ virtual std::auto_ptr<ProtocolObject> StartCommand(BackupProtocolObject&
+ rCommand) = 0;
+ virtual ~TestHook() { }
+ };
+ void SetTestHook(TestHook& rTestHook)
+ {
+ mpTestHook = &rTestHook;
+ }
+ std::auto_ptr<ProtocolObject> StartCommandHook(BackupProtocolObject& rCommand)
+ {
+ if(mpTestHook)
+ {
+ return mpTestHook->StartCommand(rCommand);
+ }
+ return std::auto_ptr<ProtocolObject>();
+ }
+
+private:
+ TestHook* mpTestHook;
+};
+
+#endif // BACKUPCONTEXT__H
+
Copied: box/trunk/lib/backupstore/BackupStoreDirectory.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreDirectory.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreDirectory.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreDirectory.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,568 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include "BackupStoreDirectory.h"
+#include "IOStream.h"
+#include "BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mNumEntries;
+ int64_t mObjectID; // this object ID
+ int64_t mContainerID; // ID of container
+ uint64_t mAttributesModTime;
+ int32_t mOptionsPresent; // bit mask of optional sections / features present
+ // Then a StreamableMemBlock for attributes
+} dir_StreamFormat;
+
+typedef enum
+{
+ Option_DependencyInfoPresent = 1
+} dir_StreamFormatOptions;
+
+typedef struct
+{
+ uint64_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ uint64_t mAttributesHash;
+ int16_t mFlags; // order smaller items after bigger ones (for alignment)
+ // Then a BackupStoreFilename
+ // Then a StreamableMemBlock for attributes
+} en_StreamFormat;
+
+typedef struct
+{
+ int64_t mDependsNewer;
+ int64_t mDependsOlder;
+} en_StreamFormatDepends;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::BackupStoreDirectory()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory()
+ : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t)
+// Purpose: Constructor giving object and container IDs
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID)
+ : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::~BackupStoreDirectory()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::~BackupStoreDirectory()
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ delete (*i);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int)
+// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Get the header
+ dir_StreamFormat hdr;
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic value...
+ if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat)
+ }
+
+ // Get data
+ mObjectID = box_ntoh64(hdr.mObjectID);
+ mContainerID = box_ntoh64(hdr.mContainerID);
+ mAttributesModTime = box_ntoh64(hdr.mAttributesModTime);
+
+ // Options
+ int32_t options = ntohl(hdr.mOptionsPresent);
+
+ // Get attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Decode count
+ int count = ntohl(hdr.mNumEntries);
+
+ // Clear existing list
+ for(std::vector<Entry*>::iterator i = mEntries.begin();
+ i != mEntries.end(); i++)
+ {
+ delete (*i);
+ }
+ mEntries.clear();
+
+ // Read them in!
+ for(int c = 0; c < count; ++c)
+ {
+ Entry *pen = new Entry;
+ try
+ {
+ // Read from stream
+ pen->ReadFromStream(rStream, Timeout);
+
+ // Add to list
+ mEntries.push_back(pen);
+ }
+ catch(...)
+ {
+ delete pen;
+ throw;
+ }
+ }
+
+ // Read in dependency info?
+ if(options & Option_DependencyInfoPresent)
+ {
+ // Read in extra dependency data
+ for(int c = 0; c < count; ++c)
+ {
+ mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool)
+// Purpose: Writes a selection of entries to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const
+{
+ // Get count of entries
+ int32_t count = mEntries.size();
+ if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Need to count the entries
+ count = 0;
+ Iterator i(*this);
+ while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0)
+ {
+ count++;
+ }
+ }
+
+ // Check that sensible IDs have been set
+ ASSERT(mObjectID != 0);
+ ASSERT(mContainerID != 0);
+
+ // Need dependency info?
+ bool dependencyInfoRequired = false;
+ if(StreamDependencyInfo)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ if(pen->HasDependencies())
+ {
+ dependencyInfoRequired = true;
+ }
+ }
+ }
+
+ // Options
+ int32_t options = 0;
+ if(dependencyInfoRequired) options |= Option_DependencyInfoPresent;
+
+ // Build header
+ dir_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE);
+ hdr.mNumEntries = htonl(count);
+ hdr.mObjectID = box_hton64(mObjectID);
+ hdr.mContainerID = box_hton64(mContainerID);
+ hdr.mAttributesModTime = box_hton64(mAttributesModTime);
+ hdr.mOptionsPresent = htonl(options);
+
+ // Write header
+ rStream.Write(&hdr, sizeof(hdr));
+
+ // Write the attributes?
+ if(StreamAttributes)
+ {
+ mAttributes.WriteToStream(rStream);
+ }
+ else
+ {
+ // Write a blank header instead
+ StreamableMemBlock::WriteEmptyBlockToStream(rStream);
+ }
+
+ // Then write all the entries
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStream(rStream);
+ }
+
+ // Write dependency info?
+ if(dependencyInfoRequired)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStreamDependencyInfo(rStream);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const Entry &)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy)
+{
+ Entry *pnew = new Entry(rEntryToCopy);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::DeleteEntry(int64_t)
+// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::DeleteEntry(int64_t ObjectID)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Delete
+ delete (*i);
+ // Remove from list
+ mEntries.erase(i);
+ // Done
+ return;
+ }
+ }
+
+ // Not found
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::FindEntryByID(int64_t)
+// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist.
+// Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const
+{
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Found
+ return (*i);
+ }
+ }
+
+ // Not found
+ return 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry()
+ : mModificationTime(0),
+ mObjectID(0),
+ mSizeInBlocks(0),
+ mFlags(0),
+ mAttributesHash(0),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const Entry &rToCopy)
+ : mName(rToCopy.mName),
+ mModificationTime(rToCopy.mModificationTime),
+ mObjectID(rToCopy.mObjectID),
+ mSizeInBlocks(rToCopy.mSizeInBlocks),
+ mFlags(rToCopy.mFlags),
+ mAttributesHash(rToCopy.mAttributesHash),
+ mAttributes(rToCopy.mAttributes),
+ mMinMarkNumber(rToCopy.mMinMarkNumber),
+ mMarkNumber(rToCopy.mMarkNumber),
+ mDependsNewer(rToCopy.mDependsNewer),
+ mDependsOlder(rToCopy.mDependsOlder)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Constructor from values
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash)
+ : mName(rName),
+ mModificationTime(ModificationTime),
+ mObjectID(ObjectID),
+ mSizeInBlocks(SizeInBlocks),
+ mFlags(Flags),
+ mAttributesHash(AttributesHash),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int)
+// Purpose: Read an entry from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormat entry;
+ if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Do reading first before modifying the variables, to be more exception safe
+
+ // Get the filename
+ BackupStoreFilename name;
+ name.ReadFromStream(rStream, Timeout);
+
+ // Get the attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Store the rest of the bits
+ mModificationTime = box_ntoh64(entry.mModificationTime);
+ mObjectID = box_ntoh64(entry.mObjectID);
+ mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks);
+ mAttributesHash = box_ntoh64(entry.mAttributesHash);
+ mFlags = ntohs(entry.mFlags);
+ mName = name;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &)
+// Purpose: Writes the entry to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const
+{
+ // Build a structure
+ en_StreamFormat entry;
+ entry.mModificationTime = box_hton64(mModificationTime);
+ entry.mObjectID = box_hton64(mObjectID);
+ entry.mSizeInBlocks = box_hton64(mSizeInBlocks);
+ entry.mAttributesHash = box_hton64(mAttributesHash);
+ entry.mFlags = htons(mFlags);
+
+ // Write it
+ rStream.Write(&entry, sizeof(entry));
+
+ // Write the filename
+ mName.WriteToStream(rStream);
+
+ // Write any attributes
+ mAttributes.WriteToStream(rStream);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int)
+// Purpose: Read the optional dependency info from a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormatDepends depends;
+ if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Store the data
+ mDependsNewer = box_ntoh64(depends.mDependsNewer);
+ mDependsOlder = box_ntoh64(depends.mDependsOlder);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &)
+// Purpose: Write the optional dependency info to a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const
+{
+ // Build structure
+ en_StreamFormatDepends depends;
+ depends.mDependsNewer = box_hton64(mDependsNewer);
+ depends.mDependsOlder = box_hton64(mDependsOlder);
+ // Write
+ rStream.Write(&depends, sizeof(depends));
+}
+
+
+
Copied: box/trunk/lib/backupstore/BackupStoreDirectory.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreDirectory.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreDirectory.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreDirectory.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,285 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDIRECTORY__H
+#define BACKUPSTOREDIRECTORY__H
+
+#include <string>
+#include <vector>
+
+#include "BackupStoreFilenameClear.h"
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+class IOStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreDirectory
+// Purpose: In memory representation of a directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreDirectory
+{
+public:
+ BackupStoreDirectory();
+ BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID);
+private:
+ // Copying not allowed
+ BackupStoreDirectory(const BackupStoreDirectory &rToCopy);
+public:
+ ~BackupStoreDirectory();
+
+ class Entry
+ {
+ public:
+ friend class BackupStoreDirectory;
+
+ Entry();
+ ~Entry();
+ Entry(const Entry &rToCopy);
+ Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash);
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ const BackupStoreFilename &GetName() const {return mName;}
+ box_time_t GetModificationTime() const {return mModificationTime;}
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetSizeInBlocks() const {return mSizeInBlocks;}
+ int16_t GetFlags() const {return mFlags;}
+ void AddFlags(int16_t Flags) {mFlags |= Flags;}
+ void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;}
+
+ // Some things can be changed
+ void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;}
+ void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ uint64_t GetAttributesHash() const {return mAttributesHash;}
+
+ // Marks
+ // The lowest mark number a version of a file of this name has ever had
+ uint32_t GetMinMarkNumber() const {return mMinMarkNumber;}
+ // The mark number on this file
+ uint32_t GetMarkNumber() const {return mMarkNumber;}
+
+ // Make sure these flags are synced with those in backupprocotol.txt
+ // ListDirectory command
+ enum
+ {
+ Flags_INCLUDE_EVERYTHING = -1,
+ Flags_EXCLUDE_NOTHING = 0,
+ Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below!
+ Flags_File = 1,
+ Flags_Dir = 2,
+ Flags_Deleted = 4,
+ Flags_OldVersion = 8,
+ Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion
+ };
+ // characters for textual listing of files -- see bbackupquery/BackupQueries
+ #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR"
+
+ // convenience methods
+ bool inline IsDir()
+ {
+ return GetFlags() & Flags_Dir;
+ }
+ bool inline IsFile()
+ {
+ return GetFlags() & Flags_File;
+ }
+ bool inline IsOld()
+ {
+ return GetFlags() & Flags_OldVersion;
+ }
+ bool inline IsDeleted()
+ {
+ return GetFlags() & Flags_Deleted;
+ }
+ bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+ {
+ return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet))
+ && ((mFlags & FlagsNotToBeSet) == 0);
+ };
+
+ // Get dependency info
+ // new version this depends on
+ int64_t GetDependsNewer() const {return mDependsNewer;}
+ void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;}
+ // older version which depends on this
+ int64_t GetDependsOlder() const {return mDependsOlder;}
+ void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;}
+
+ // Dependency info saving
+ bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;}
+ void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout);
+ void WriteToStreamDependencyInfo(IOStream &rStream) const;
+
+ private:
+ BackupStoreFilename mName;
+ box_time_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ int16_t mFlags;
+ uint64_t mAttributesHash;
+ StreamableMemBlock mAttributes;
+ uint32_t mMinMarkNumber;
+ uint32_t mMarkNumber;
+
+ uint64_t mDependsNewer; // new version this depends on
+ uint64_t mDependsOlder; // older version which depends on this
+ };
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream,
+ int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING,
+ int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING,
+ bool StreamAttributes = true, bool StreamDependencyInfo = true) const;
+
+ Entry *AddEntry(const Entry &rEntryToCopy);
+ Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime);
+ void DeleteEntry(int64_t ObjectID);
+ Entry *FindEntryByID(int64_t ObjectID) const;
+
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetContainerID() const {return mContainerID;}
+
+ // Need to be able to update the container ID when moving objects
+ void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;}
+
+ // Purely for use of server -- not serialised into streams
+ int64_t GetRevisionID() const {return mRevisionID;}
+ void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;}
+
+ unsigned int GetNumberOfEntries() const {return mEntries.size();}
+
+ // User info -- not serialised into streams
+ int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;}
+ void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ box_time_t GetAttributesModTime() const {return mAttributesModTime;}
+
+ class Iterator
+ {
+ public:
+ Iterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.begin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ // WARNING: This function is really very inefficient.
+ // Only use when you want to look up ONE filename, not in a loop looking up lots.
+ // In a looping situation, cache the decrypted filenames in another memory structure.
+ BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags or filename
+ while( (i != mrDir.mEntries.end())
+ && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) )
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_iterator i;
+ };
+
+ friend class Iterator;
+
+ class ReverseIterator
+ {
+ public:
+ ReverseIterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.rbegin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.rend())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_reverse_iterator i;
+ };
+
+ friend class ReverseIterator;
+
+ // For recovery of the store
+ // Implemented in BackupStoreCheck2.cpp
+ bool CheckAndFix();
+ void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags);
+ bool NameInUse(const BackupStoreFilename &rName);
+ // Don't use these functions in normal code!
+
+ // For testing
+ void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;}
+
+ // Debug and diagonistics
+ void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere
+
+private:
+ int64_t mRevisionID;
+ int64_t mObjectID;
+ int64_t mContainerID;
+ std::vector<Entry*> mEntries;
+ box_time_t mAttributesModTime;
+ StreamableMemBlock mAttributes;
+ int64_t mUserInfo1;
+};
+
+#endif // BACKUPSTOREDIRECTORY__H
+
Copied: box/trunk/lib/backupstore/BackupStoreException.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreException.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreException.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreException.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreException.h
+// Purpose: Exception
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREEXCEPTION__H
+#define BACKUPSTOREEXCEPTION__H
+
+// Compatibility
+#include "autogen_BackupStoreException.h"
+
+#endif // BACKUPSTOREEXCEPTION__H
+
Copied: box/trunk/lib/backupstore/BackupStoreException.txt (from rev 2943, box/trunk/lib/backupclient/BackupStoreException.txt)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreException.txt (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreException.txt 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,72 @@
+EXCEPTION BackupStore 4
+
+Internal 0
+BadAccountDatabaseFile 1
+AccountDatabaseNoSuchEntry 2
+InvalidBackupStoreFilename 3
+UnknownFilenameEncoding 4
+CouldntReadEntireStructureFromStream 5
+BadDirectoryFormat 6
+CouldNotFindEntryInDirectory 7
+OutputFileAlreadyExists 8
+OSFileError 9
+StreamDoesntHaveRequiredFeatures 10
+BadBackupStoreFile 11
+CouldNotLoadStoreInfo 12
+BadStoreInfoOnLoad 13
+StoreInfoIsReadOnly 14
+StoreInfoDirNotInList 15
+StoreInfoBlockDeltaMakesValueNegative 16
+DirectoryHasBeenDeleted 17
+StoreInfoNotInitialised 18
+StoreInfoAlreadyLoaded 19
+StoreInfoNotLoaded 20
+ReadFileFromStreamTimedOut 21
+FileWrongSizeAfterBeingStored 22
+AddedFileDoesNotVerify 23
+StoreInfoForWrongAccount 24
+ContextIsReadOnly 25
+AttributesNotLoaded 26
+AttributesNotUnderstood 27
+WrongServerVersion 28 # client side
+ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account.
+NameAlreadyExistsInDirectory 30
+BerkelyDBFailure 31 # client side
+InodeMapIsReadOnly 32 # client side
+InodeMapNotOpen 33 # client side
+FilenameEncryptionKeyNotKnown 34
+FilenameEncryptionNoKeyForSpecifiedMethod 35
+FilenameEncryptionNotSetup 36
+CouldntLoadClientKeyMaterial 37
+BadEncryptedAttributes 38
+EncryptedAttributesHaveUnknownEncoding 39
+OutputSizeTooSmallForChunk 40
+BadEncodedChunk 41
+NotEnoughSpaceToDecodeChunk 42
+ChunkHasUnknownEncoding 43
+ChunkContainsBadCompressedData 44
+CantWriteToEncodedFileStream 45
+Temp_FileEncodeStreamDidntReadBuffer 46
+CantWriteToDecodedFileStream 47
+WhenDecodingExpectedToReadButCouldnt 48
+BackupStoreFileFailedIntegrityCheck 49
+ThereIsNoDataInASymLink 50
+IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51
+BlockEntryEncodingDidntGiveExpectedLength 52
+CouldNotFindUnusedIDDuringAllocation 53
+AddedFileExceedsStorageLimit 54
+CannotDiffAnIncompleteStoreFile 55
+CannotDecodeDiffedFilesWithoutCombining 56
+FailedToReadBlockOnCombine 57
+OnCombineFromFileIsIncomplete 58
+BadNotifySysadminEventCode 59
+InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60
+CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server?
+AttributeHashSecretNotSet 62
+AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES
+SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection.
+IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related
+DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory
+PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
+UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known.
+MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first
Copied: box/trunk/lib/backupstore/BackupStoreFile.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreFile.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFile.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFile.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,1559 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.cpp
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include <sys/stat.h>
+#include <string.h>
+#include <new>
+#include <string.h>
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ #include <stdio.h>
+#endif
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFilename.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+#include "FileModificationTime.h"
+#include "FileStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+#include "Compress.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "CipherAES.h"
+#include "BackupStoreConstants.h"
+#include "CollectInBufferStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "ReadGatherStream.h"
+#include "Random.h"
+#include "BackupStoreFileEncodeStream.h"
+#include "Logging.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+// How big a buffer to use for copying files
+#define COPY_BUFFER_SIZE (8*1024)
+
+// Statistics
+BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool sWarnedAboutBackwardsCompatiblity = false;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &)
+// Purpose: Encode a file into something for storing on file server.
+// Requires a real filename so full info can be stored.
+//
+// Returns a stream. Most of the work is done by the stream
+// when data is actually requested -- the file will be held
+// open until the stream is deleted or the file finished.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(
+ const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename,
+ int64_t *pModificationTime,
+ ReadLoggingStream::Logger* pLogger,
+ RunStatusProvider* pRunStatusProvider)
+{
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename,
+ 0 /* no recipe, just encode */,
+ ContainerID, rStoreFilename, pModificationTime, pLogger,
+ pRunStatusProvider);
+
+ // Return the stream for the caller
+ return stream;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
+// Purpose: Verify that an encoded file meets the format
+// requirements. Doesn't verify that the data is intact
+// and can be decoded. Optionally returns the ID of the
+// file which it is diffed from, and the (original)
+// container ID.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
+{
+ // Get the size of the file
+ int64_t fileSize = rFile.BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Get the header...
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read header
+ return false;
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ return false;
+ }
+
+ // Get a filename, see if it loads OK
+ try
+ {
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
+ try
+ {
+ int32_t size_s;
+ if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFile.Seek(size, IOStream::SeekType_Relative);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Get current position in file -- the end of the header
+ int64_t headerEnd = rFile.GetPosition();
+
+ // Get number of blocks
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ if(blockIndexLoc < headerEnd)
+ {
+ // Not enough space left for the block index, let alone the blocks themselves
+ return false;
+ }
+
+ // Load the block index header
+ rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
+ file_BlockIndexHeader blkhdr;
+ if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index header -- assume bad file
+ return false;
+ }
+
+ // Check header
+ if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks)
+ {
+ // Bad header -- either magic value or number of blocks is wrong
+ return false;
+ }
+
+ // Flag for recording whether a block is referenced from another file
+ bool blockFromOtherFileReferenced = false;
+
+ // Read the index, checking that the length values all make sense
+ int64_t currentBlockStart = headerEnd;
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read block entry
+ file_BlockIndexEntry blk;
+ if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index entry -- assume bad file
+ return false;
+ }
+
+ // Check size and location
+ int64_t blkSize = box_ntoh64(blk.mEncodedSize);
+ if(blkSize <= 0)
+ {
+ // Mark that this file references another file
+ blockFromOtherFileReferenced = true;
+ }
+ else
+ {
+ // This block is actually in this file
+ if((currentBlockStart + blkSize) > blockIndexLoc)
+ {
+ // Encoded size makes the block run over the index
+ return false;
+ }
+
+ // Move the current block start ot the end of this block
+ currentBlockStart += blkSize;
+ }
+ }
+
+ // Check that there's no empty space
+ if(currentBlockStart != blockIndexLoc)
+ {
+ return false;
+ }
+
+ // Check that if another block is references, then the ID is there, and if one isn't there is no ID.
+ int64_t otherID = box_ntoh64(blkhdr.mOtherFileID);
+ if((otherID != 0 && blockFromOtherFileReferenced == false)
+ || (otherID == 0 && blockFromOtherFileReferenced == true))
+ {
+ // Doesn't look good!
+ return false;
+ }
+
+ // Does the caller want the other ID?
+ if(pDiffFromObjectIDOut)
+ {
+ *pDiffFromObjectIDOut = otherID;
+ }
+
+ // Does the caller want the container ID?
+ if(pContainerIDOut)
+ {
+ *pContainerIDOut = box_ntoh64(hdr.mContainerID);
+ }
+
+ // Passes all tests
+ return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFile(IOStream &, const char *)
+// Purpose: Decode a file. Will set file attributes. File must not exist.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Does file exist?
+ EMU_STRUCT_STAT st;
+ if(EMU_STAT(DecodedFilename, &st) == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
+ }
+
+ // Try, delete output file if error
+ try
+ {
+ // Make a stream for outputting this file
+ FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
+
+ // Get the decoding stream
+ std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
+
+ // Is it a symlink?
+ if(!stream->IsSymLink())
+ {
+ // Copy it out to the file
+ stream->CopyStreamTo(out);
+ }
+
+ out.Close();
+
+ // The stream might have uncertain size, in which case
+ // we need to drain it to get the
+ // Protocol::ProtocolStreamHeader_EndOfStream byte
+ // out of our connection stream.
+ char buffer[1];
+ int drained = rEncodedFile.Read(buffer, 1);
+
+ // The Read will return 0 if we are actually at the end
+ // of the stream, but some tests decode files directly,
+ // in which case we are actually positioned at the start
+ // of the block index. I hope that reading an extra byte
+ // doesn't hurt!
+ // ASSERT(drained == 0);
+
+ // Write the attributes
+ try
+ {
+ stream->GetAttributes().WriteAttributes(DecodedFilename);
+ }
+ catch (std::exception& e)
+ {
+ BOX_WARNING("Failed to restore attributes on " <<
+ DecodedFilename << ": " << e.what());
+ }
+ }
+ catch(...)
+ {
+ ::unlink(DecodedFilename);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
+// Purpose: Return a stream which will decode the encrypted file data on the fly.
+// Accepts streams in block index first, or main header first, order. In the latter case,
+// the stream must be Seek()able.
+//
+// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
+// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Create stream
+ std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
+
+ // Get it ready
+ stream->Setup(pAlterativeAttr);
+
+ // Return to caller
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
+// Purpose: Constructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
+ : mrEncodedFile(rEncodedFile),
+ mTimeout(Timeout),
+ mNumBlocks(0),
+ mpBlockIndex(0),
+ mpEncodedData(0),
+ mpClearData(0),
+ mClearDataSize(0),
+ mCurrentBlock(-1),
+ mCurrentBlockClearSize(0),
+ mPositionInCurrentBlock(0),
+ mEntryIVBase(42) // different to default value in the encoded stream!
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ , mIsOldVersion(false)
+#endif
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::~DecodedStream()
+// Purpose: Desctructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::~DecodedStream()
+{
+ // Free any allocated memory
+ if(mpBlockIndex)
+ {
+ ::free(mpBlockIndex);
+ }
+ if(mpEncodedData)
+ {
+ BackupStoreFile::CodingChunkFree(mpEncodedData);
+ }
+ if(mpClearData)
+ {
+ ::free(mpClearData);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
+// Purpose: Get the stream ready to decode -- reads in headers
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Get the size of the file
+ int64_t fileSize = mrEncodedFile.BytesLeftToRead();
+
+ // Get the magic number to work out which order the stream is in
+ int32_t magic;
+ if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read magic value
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ bool inFileOrder = true;
+ switch(ntohl(magic))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+ inFileOrder = true;
+ break;
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
+ inFileOrder = false;
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // If not in file order, then the index list must be read now
+ if(!inFileOrder)
+ {
+ ReadBlockIndex(true /* have already read and verified the magic number */);
+ }
+
+ // Get header
+ file_StreamFormat hdr;
+ if(inFileOrder)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ // Put in magic number
+ hdr.mMagicValue = magic;
+ }
+ else
+ {
+ // Not in file order, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get the filename
+ mFilename.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Get the attributes (either from stream, or supplied attributes)
+ if(pAlterativeAttr != 0)
+ {
+ // Read dummy attributes
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Set to supplied attributes
+ mAttributes = *pAlterativeAttr;
+ }
+ else
+ {
+ // Read the attributes from the stream
+ mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
+ }
+
+ // If it is in file order, go and read the file attributes
+ // Requires that the stream can seek
+ if(inFileOrder)
+ {
+ // Make sure the file size is known
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Store current location (beginning of encoded blocks)
+ int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
+
+ // Work out where the index is
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Seek to that position
+ mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
+
+ // Read the block index
+ ReadBlockIndex(false /* magic number still to be read */);
+
+ // Seek back to the end of header position, ready for reading the chunks
+ mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
+ }
+
+ // Check view of blocks from block header and file header match
+ if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Need to allocate some memory for the two blocks for reading encoded data, and clear data
+ if(mNumBlocks > 0)
+ {
+ // Find the maximum encoded data size
+ int32_t maxEncodedDataSize = 0;
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ ASSERT(entry != 0);
+ for(int64_t e = 0; e < mNumBlocks; e++)
+ {
+ // Get the clear and encoded size
+ int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize);
+ ASSERT(encodedSize > 0);
+
+ // Larger?
+ if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
+ }
+
+ // Allocate those blocks!
+ mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
+
+ // Allocate the block for the clear data, using the hint from the header.
+ // If this is wrong, things will exception neatly later on, so it can't be used
+ // to do anything more than cause an error on downloading.
+ mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
+ mpClearData = (uint8_t*)::malloc(mClearDataSize);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
+// Purpose: Read the block index from the stream, and store in internal buffer (minus header)
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
+{
+ // Header
+ file_BlockIndexHeader blkhdr;
+
+ // Read it in -- way depends on how whether the magic number has already been read
+ if(MagicAlreadyRead)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+ else
+ {
+ // Magic not already read, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check magic value
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ }
+
+ // Get the number of blocks out of the header
+ mNumBlocks = box_ntoh64(blkhdr.mNumBlocks);
+
+ // Read the IV base
+ mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase);
+
+ // Load the block entries in?
+ if(mNumBlocks > 0)
+ {
+ // How big is the index?
+ int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
+
+ // Allocate some memory
+ mpBlockIndex = ::malloc(indexSize);
+ if(mpBlockIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Read it in
+ if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Read(void *, int, int)
+// Purpose: As interface. Reads decrpyted data.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Symlinks don't have data. So can't read it. Not even zero bytes.
+ if(IsSymLink())
+ {
+ // Don't allow reading in this case
+ THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
+ }
+
+ // Already finished?
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // At end of stream, nothing to do
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *output = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
+ {
+ // Anything left in the current block?
+ if(mPositionInCurrentBlock < mCurrentBlockClearSize)
+ {
+ // Copy data out of this buffer
+ int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead; // limit to requested data
+
+ // Copy
+ ::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
+
+ // Update positions
+ output += s;
+ mPositionInCurrentBlock += s;
+ bytesToRead -= s;
+ }
+
+ // Need to get some more data?
+ if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
+ {
+ // Number of next block
+ ++mCurrentBlock;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Stop now!
+ break;
+ }
+
+ // Get the size from the block index
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize);
+ if(encodedSize <= 0)
+ {
+ // The caller is attempting to decode a file which is the direct result of a diff
+ // operation, and so does not contain all the data.
+ // It needs to be combined with the previous version first.
+ THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
+ }
+
+ // Load in next block
+ if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Decode the data
+ mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
+
+ // Calculate IV for this entry
+ uint64_t iv = mEntryIVBase;
+ iv += mCurrentBlock;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = box_hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Make sure this is the right size
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(!mIsOldVersion)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
+ // IV for the encrypted section. Try again, with the thing the other way round
+ iv = box_swap64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ else
+ {
+ // Warn and log this issue
+ if(!sWarnedAboutBackwardsCompatiblity)
+ {
+ BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index.");
+ sWarnedAboutBackwardsCompatiblity = true;
+ }
+ }
+#else
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+#endif
+ }
+
+ // Check the digest
+ MD5Digest md5;
+ md5.Add(mpClearData, mCurrentBlockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
+ {
+ THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
+ }
+
+ // Set vars to say what's happening
+ mPositionInCurrentBlock = 0;
+ }
+ }
+
+ ASSERT(bytesToRead >= 0);
+ ASSERT(bytesToRead <= NBytes);
+
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::IsSymLink()
+// Purpose: Is the unencoded file actually a symlink?
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::IsSymLink()
+{
+ // First, check in with the attributes
+ if(!mAttributes.IsSymLink())
+ {
+ return false;
+ }
+
+ // So the attributes think it is a symlink.
+ // Consistency check...
+ if(mNumBlocks != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Write(const void *, int)
+// Purpose: As interface. Throws exception, as you can't write to this stream.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamDataLeft()
+// Purpose: As interface. Any data left?
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamDataLeft()
+{
+ return mCurrentBlock < mNumBlocks;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamClosed()
+// Purpose: As interface. Always returns true, no writing allowed.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamClosed()
+{
+ // Can't write to this stream!
+ return true;
+}
+
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ sBlowfishEncryptBlockEntry.Reset();
+ sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishEncryptBlockEntry.UsePadding(false);
+ sBlowfishDecryptBlockEntry.Reset();
+ sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishDecryptBlockEntry.UsePadding(false);
+}
+
+
+#ifndef HAVE_OLD_SSL
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetAESKey(const void *, int)
+// Purpose: Sets the AES key to use for file data encryption. Will select AES as
+// the cipher to use when encrypting.
+// Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
+{
+ // Setup context
+ sAESEncrypt.Reset();
+ sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sAESDecrypt.Reset();
+ sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ // Set encryption to use this key, instead of the "default" blowfish key
+ spEncrypt = &sAESEncrypt;
+ sEncryptCipherType = HEADER_AES_ENCODING;
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int)
+// Purpose: The maximum output size of a block, given the chunk size
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
+{
+ // Calculate... the maximum size of output by first the largest it could be after compression,
+ // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
+ // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
+ // an problem where a rather non-compressable file didn't fit in a block buffer.)
+ return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
+ + sBlowfishEncrypt.GetIVLength() + 128;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
+// Purpose: Encodes a chunk (encryption, possible compressed beforehand)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
+{
+ ASSERT(spEncrypt != 0);
+
+ // Check there's some space in the output block
+ if(rOutput.mBufferSize < 256)
+ {
+ rOutput.Reallocate(256);
+ }
+
+ // Check alignment of the block
+ ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // Want to compress it?
+ bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
+
+ // Build header
+ uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
+ if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
+
+ // Store header
+ rOutput.mpBuffer[0] = header;
+ int outOffset = 1;
+
+ // Setup cipher, and store the IV
+ int ivLen = 0;
+ const void *iv = spEncrypt->SetRandomIV(ivLen);
+ ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
+ outOffset += ivLen;
+
+ // Start encryption process
+ spEncrypt->Begin();
+
+ #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \
+ { \
+ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \
+ { \
+ rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \
+ } \
+ }
+
+ // Encode the chunk
+ if(compressChunk)
+ {
+ // buffer to compress into
+ uint8_t buffer[2048];
+
+ // Set compressor with all the chunk as an input
+ Compress<true> compress;
+ compress.Input(Chunk, ChunkSize);
+ compress.FinishInput();
+
+ // Get and encrypt output
+ while(!compress.OutputHasFinished())
+ {
+ int s = compress.Output(buffer, sizeof(buffer));
+ if(s > 0)
+ {
+ ENCODECHUNK_CHECK_SPACE(s)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);
+ }
+ else
+ {
+ // Should never happen, as we put all the input in in one go.
+ // So if this happens, it means there's a logical problem somewhere
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ }
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+ else
+ {
+ // Straight encryption
+ ENCODECHUNK_CHECK_SPACE(ChunkSize)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+
+ ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check
+
+ return outOffset;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int)
+// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
+// the extra output buffer size needed before calling.
+// See notes in EncodeChunk() for notes re alignment of the
+// encoded data.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
+{
+ // Check alignment of the encoded block
+ ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // First check
+ if(EncodedSize < 1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ const uint8_t *input = (uint8_t*)Encoded;
+
+ // Get header, make checks, etc
+ uint8_t header = input[0];
+ bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
+ uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
+ if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
+ }
+
+#ifndef HAVE_OLD_SSL
+ // Choose cipher
+ CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
+#else
+ // AES not supported with this version of OpenSSL
+ if(encodingType == HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
+ }
+ CipherContext &cipher(sBlowfishDecrypt);
+#endif
+
+ // Check enough space for header, an IV and one byte of input
+ int ivLen = cipher.GetIVLength();
+ if(EncodedSize < (1 + ivLen + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ // Set IV in decrypt context, and start
+ cipher.SetIV(input + 1);
+ cipher.Begin();
+
+ // Setup vars for code
+ int inOffset = 1 + ivLen;
+ uint8_t *output = (uint8_t*)Output;
+ int outOffset = 0;
+
+ // Do action
+ if(chunkCompressed)
+ {
+ // Do things in chunks
+ uint8_t buffer[2048];
+ int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
+
+ // Decompressor
+ Compress<false> decompress;
+
+ while(inOffset < EncodedSize)
+ {
+ // Decrypt a block
+ int bl = inputBlockLen;
+ if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long
+ int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
+ inOffset += bl;
+
+ // Decompress the decrypted data
+ if(s > 0)
+ {
+ decompress.Input(buffer, s);
+ int os = 0;
+ do
+ {
+ os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+ } while(os > 0);
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+
+ // Get any compressed data remaining in the cipher context and compression
+ int s = cipher.Final(buffer, sizeof(buffer));
+ decompress.Input(buffer, s);
+ decompress.FinishInput();
+ while(!decompress.OutputHasFinished())
+ {
+ int os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+ else
+ {
+ // Easy decryption
+ outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
+ outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
+ }
+
+ return outOffset;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
+// Purpose: Returns a stream which gives a Stream order version of the encoded file.
+// If TakeOwnership == true, then the input stream will be deleted when the
+// returned stream is deleted.
+// The input stream must be seekable.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
+{
+ ASSERT(pStream != 0);
+
+ // Get the size of the file
+ int64_t fileSize = pStream->BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Read the header
+ int bytesRead = 0;
+ file_StreamFormat hdr;
+ bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
+
+ // Seek backwards to put the file pointer back where it was before we started this
+ pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
+
+ // Check we got a block
+ if(!readBlock)
+ {
+ // Couldn't read header -- assume file bad
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get number of blocks
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ int64_t blockIndexLoc = fileSize - blockIndexSize;
+ if(blockIndexLoc < 0)
+ {
+ // Doesn't look good!
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Build a reordered stream
+ std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
+
+ // Set it up...
+ ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
+ int component = rreordered.AddComponent(pStream);
+ // Send out the block index
+ rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
+ // And then the rest of the file
+ rreordered.AddBlock(component, blockIndexLoc, true, 0);
+
+ return reordered;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ResetStats()
+// Purpose: Reset the gathered statistics
+// Created: 20/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ResetStats()
+{
+ msStats.mBytesInEncodedFiles = 0;
+ msStats.mBytesAlreadyOnServer = 0;
+ msStats.mTotalFileStreamSize = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
+// Purpose: Compares the contents of a file against the checksums contained in the
+// block index. Returns true if the checksums match, meaning the file is
+// extremely likely to match the original. Will always consume the entire index.
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
+{
+ // is it a symlink?
+ bool sourceIsSymlink = false;
+ {
+ EMU_STRUCT_STAT st;
+ if(EMU_LSTAT(Filename, &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ sourceIsSymlink = true;
+ }
+ }
+
+ // Open file, if it's not a symlink
+ std::auto_ptr<FileStream> in;
+ if(!sourceIsSymlink)
+ {
+ in.reset(new FileStream(Filename));
+ }
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
+#endif
+
+ // Get basic information
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // setup
+ void *data = 0;
+ int32_t dataSize = -1;
+ bool matches = true;
+ int64_t totalSizeInBlockIndex = 0;
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ iv = box_hton64(iv);
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(isOldVersion)
+ {
+ // Reverse the IV for compatibility
+ iv = box_swap64(iv);
+ }
+#endif
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Size of block
+ int32_t blockClearSize = ntohl(entryEnc.mSize);
+ if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ totalSizeInBlockIndex += blockClearSize;
+
+ // Make sure there's enough memory allocated to load the block in
+ if(dataSize < blockClearSize)
+ {
+ // Too small, free the block if it's already allocated
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ // Allocate a block
+ data = ::malloc(blockClearSize + 128);
+ if(data == 0)
+ {
+ throw std::bad_alloc();
+ }
+ dataSize = blockClearSize + 128;
+ }
+
+ // Load in the block from the file, if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ if(in->Read(data, blockClearSize) != blockClearSize)
+ {
+ // Not enough data left in the file, can't possibly match
+ matches = false;
+ }
+ else
+ {
+ // Check the checksum
+ MD5Digest md5;
+ md5.Add(data, blockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches(entryEnc.mStrongChecksum))
+ {
+ // Checksum didn't match
+ matches = false;
+ }
+ }
+ }
+
+ // Keep on going regardless, to make sure the entire block index stream is read
+ // -- must always be consistent about what happens with the stream.
+ }
+ }
+ catch(...)
+ {
+ // clean up in case of errors
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ throw;
+ }
+
+ // free block
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+
+ // Check for data left over if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ // Anything left to read in the file?
+ if(in->BytesLeftToRead() != 0)
+ {
+ // File has extra data at the end
+ matches = false;
+ }
+ }
+
+ // Symlinks must have zero size on server
+ if(sourceIsSymlink)
+ {
+ matches = (totalSizeInBlockIndex == 0);
+ }
+
+ return matches;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer()
+// Purpose: Constructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::EncodingBuffer()
+ : mpBuffer(0),
+ mBufferSize(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+// Purpose: Destructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+{
+ if(mpBuffer != 0)
+ {
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+ mpBuffer = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Allocate(int)
+// Purpose: Do initial allocation of block
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Allocate(int Size)
+{
+ ASSERT(mpBuffer == 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ mpBuffer = buffer;
+ mBufferSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Reallocate(int)
+// Purpose: Reallocate the block. Try not to call this, it has to copy
+// the entire contents as the block can't be reallocated straight.
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
+{
+ BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize <<
+ " to " << NewSize);
+ ASSERT(mpBuffer != 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Copy data
+ ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
+
+ // Free old
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+
+ // Store new buffer
+ mpBuffer = buffer;
+ mBufferSize = NewSize;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: DiffTimer::DiffTimer();
+// Purpose: Constructor
+// Created: 2005/02/01
+//
+// --------------------------------------------------------------------------
+DiffTimer::DiffTimer()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: DiffTimer::DiffTimer();
+// Purpose: Destructor
+// Created: 2005/02/01
+//
+// --------------------------------------------------------------------------
+DiffTimer::~DiffTimer()
+{
+}
Copied: box/trunk/lib/backupstore/BackupStoreFile.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreFile.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFile.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFile.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,231 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.h
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILE__H
+#define BACKUPSTOREFILE__H
+
+#include <cstdlib>
+#include <memory>
+#include <cstdlib>
+
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFilename.h"
+#include "IOStream.h"
+#include "ReadLoggingStream.h"
+#include "RunStatusProvider.h"
+
+typedef struct
+{
+ int64_t mBytesInEncodedFiles;
+ int64_t mBytesAlreadyOnServer;
+ int64_t mTotalFileStreamSize;
+} BackupStoreFileStats;
+
+// Uncomment to disable backwards compatibility
+//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+
+
+// Output buffer to EncodeChunk and input data to DecodeChunk must
+// have specific alignment, see function comments.
+#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16
+#define BACKUPSTOREFILE_CODING_OFFSET 15
+
+// Have some memory allocation commands, note closing "Off" at end of file.
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: DiffTimer
+// Purpose: Interface for classes that can keep track of diffing time,
+// and send SSL keepalive messages
+// Created: 2006/01/19
+//
+// --------------------------------------------------------------------------
+class DiffTimer
+{
+public:
+ DiffTimer();
+ virtual ~DiffTimer();
+public:
+ virtual void DoKeepAlive() = 0;
+ virtual int GetMaximumDiffingTime() = 0;
+ virtual bool IsManaged() = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFile
+// Purpose: Class to hold together utils for manipulating files.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreFile
+{
+public:
+ class DecodedStream : public IOStream
+ {
+ friend class BackupStoreFile;
+ private:
+ DecodedStream(IOStream &rEncodedFile, int Timeout);
+ DecodedStream(const DecodedStream &); // not allowed
+ DecodedStream &operator=(const DecodedStream &); // not allowed
+ public:
+ ~DecodedStream();
+
+ // Stream functions
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+ // Accessor functions
+ const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
+ const BackupStoreFilename &GetFilename() {return mFilename;}
+ int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests
+
+ bool IsSymLink();
+
+ private:
+ void Setup(const BackupClientFileAttributes *pAlterativeAttr);
+ void ReadBlockIndex(bool MagicAlreadyRead);
+
+ private:
+ IOStream &mrEncodedFile;
+ int mTimeout;
+ BackupClientFileAttributes mAttributes;
+ BackupStoreFilename mFilename;
+ int64_t mNumBlocks;
+ void *mpBlockIndex;
+ uint8_t *mpEncodedData;
+ uint8_t *mpClearData;
+ int mClearDataSize;
+ int mCurrentBlock;
+ int mCurrentBlockClearSize;
+ int mPositionInCurrentBlock;
+ uint64_t mEntryIVBase;
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool mIsOldVersion;
+#endif
+ };
+
+
+ // Main interface
+ static std::auto_ptr<IOStream> EncodeFile
+ (
+ const char *Filename,
+ int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
+ int64_t *pModificationTime = 0,
+ ReadLoggingStream::Logger* pLogger = NULL,
+ RunStatusProvider* pRunStatusProvider = NULL
+ );
+ static std::auto_ptr<IOStream> EncodeFileDiff
+ (
+ const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename,
+ int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout,
+ DiffTimer *pDiffTimer,
+ int64_t *pModificationTime = 0,
+ bool *pIsCompletelyDifferent = 0
+ );
+ static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
+ static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
+ static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
+ static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
+ static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
+ static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
+
+ // Stream manipulation
+ static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
+ static void MoveStreamPositionToBlockIndex(IOStream &rStream);
+
+ // Crypto setup
+ static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
+#ifndef HAVE_OLD_SSL
+ static void SetAESKey(const void *pKey, int KeyLength);
+#endif
+
+ // Allocation of properly aligning chunks for decoding and encoding chunks
+ inline static void *CodingChunkAlloc(int Size)
+ {
+ uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
+ if(a == 0) return 0;
+ // Align to main block size
+ ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
+ uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE
+ - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
+ uint8_t *b = (a + adjustment);
+ // Store adjustment
+ *b = adjustment;
+ // Return offset
+ return b + BACKUPSTOREFILE_CODING_OFFSET;
+ }
+ inline static void CodingChunkFree(void *Block)
+ {
+ // Check alignment is as expected
+ ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
+ ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+ uint8_t *a = (uint8_t*)Block;
+ a -= BACKUPSTOREFILE_CODING_OFFSET;
+ // Adjust downwards...
+ a -= *a;
+ free(a);
+ }
+
+ static void DiffTimerExpired();
+
+ // Building blocks
+ class EncodingBuffer
+ {
+ public:
+ EncodingBuffer();
+ ~EncodingBuffer();
+ private:
+ // No copying
+ EncodingBuffer(const EncodingBuffer &);
+ EncodingBuffer &operator=(const EncodingBuffer &);
+ public:
+ void Allocate(int Size);
+ void Reallocate(int NewSize);
+
+ uint8_t *mpBuffer;
+ int mBufferSize;
+ };
+ static int MaxBlockSizeForChunkSize(int ChunkSize);
+ static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
+
+ // Caller should know how big the output size is, but also allocate a bit more memory to cover various
+ // overheads allowed for in checks
+ static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
+ {
+ // Plenty big enough
+ return KnownChunkSize + 256;
+ }
+ static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
+
+ // Statisitics, not designed to be completely reliable
+ static void ResetStats();
+ static BackupStoreFileStats msStats;
+
+ // For debug
+#ifndef BOX_RELEASE_BUILD
+ static bool TraceDetailsOfDiffProcess;
+#endif
+
+ // For decoding encoded files
+ static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // BACKUPSTOREFILE__H
Copied: box/trunk/lib/backupstore/BackupStoreFileCryptVar.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreFileCryptVar.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFileCryptVar.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFileCryptVar.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.cpp
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFileWire.h"
+
+#include "MemLeakFindOn.h"
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt;
+
+#ifndef HAVE_OLD_SSL
+ CipherContext BackupStoreFileCryptVar::sAESEncrypt;
+ CipherContext BackupStoreFileCryptVar::sAESDecrypt;
+#endif
+
+// Default to blowfish
+CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt;
+uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING;
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry;
+
Copied: box/trunk/lib/backupstore/BackupStoreFileCryptVar.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreFileCryptVar.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFileCryptVar.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFileCryptVar.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,39 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.h
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILECRYPTVAR__H
+#define BACKUPSTOREFILECRYPTVAR__H
+
+#include "CipherContext.h"
+
+// Hide private static variables from the rest of the world by putting them
+// as static variables in a namespace.
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace BackupStoreFileCryptVar
+{
+ // Keys for the main file data
+ extern CipherContext sBlowfishEncrypt;
+ extern CipherContext sBlowfishDecrypt;
+ // Use AES when available
+#ifndef HAVE_OLD_SSL
+ extern CipherContext sAESEncrypt;
+ extern CipherContext sAESDecrypt;
+#endif
+ // How encoding will be done
+ extern CipherContext *spEncrypt;
+ extern uint8_t sEncryptCipherType;
+
+ // Keys for the block indicies
+ extern CipherContext sBlowfishEncryptBlockEntry;
+ extern CipherContext sBlowfishDecryptBlockEntry;
+}
+
+#endif // BACKUPSTOREFILECRYPTVAR__H
+
Copied: box/trunk/lib/backupstore/BackupStoreFileEncodeStream.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreFileEncodeStream.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFileEncodeStream.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFileEncodeStream.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,717 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.cpp
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BoxTime.h"
+#include "FileStream.h"
+#include "Random.h"
+#include "RollingChecksum.h"
+
+#include "MemLeakFindOn.h"
+
+#include <cstring>
+
+using namespace BackupStoreFileCryptVar;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
+// Purpose: Constructor (opens file)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
+ : mpRecipe(0),
+ mpFile(0),
+ mpLogging(0),
+ mpRunStatusProvider(NULL),
+ mStatus(Status_Header),
+ mSendData(true),
+ mTotalBlocks(0),
+ mAbsoluteBlockNumber(-1),
+ mInstructionNumber(-1),
+ mNumBlocks(0),
+ mCurrentBlock(-1),
+ mCurrentBlockEncodedSize(0),
+ mPositionInCurrentBlock(0),
+ mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
+ mLastBlockSize(0),
+ mTotalBytesSent(0),
+ mpRawBuffer(0),
+ mAllocatedBufferSize(0),
+ mEntryIVBase(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+// Purpose: Destructor
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+{
+ // Free buffers
+ if(mpRawBuffer)
+ {
+ ::free(mpRawBuffer);
+ mpRawBuffer = 0;
+ }
+
+ // Close the file, which we might have open
+ if(mpFile)
+ {
+ delete mpFile;
+ mpFile = 0;
+ }
+
+ // Clear up logging stream
+ if(mpLogging)
+ {
+ delete mpLogging;
+ mpLogging = 0;
+ }
+
+ // Free the recipe
+ if(mpRecipe != 0)
+ {
+ delete mpRecipe;
+ mpRecipe = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
+// Purpose: Reads file information, and builds file header reading for sending.
+// Takes ownership of the Recipe.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Setup(const char *Filename,
+ BackupStoreFileEncodeStream::Recipe *pRecipe,
+ int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
+ int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger,
+ RunStatusProvider* pRunStatusProvider)
+{
+ // Pointer to a blank recipe which we might create
+ BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
+
+ try
+ {
+ // Get file attributes
+ box_time_t modTime = 0;
+ int64_t fileSize = 0;
+ BackupClientFileAttributes attr;
+ attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
+ 0 /* not interested in attr mod time */, &fileSize);
+
+ // Might need to create a blank recipe...
+ if(pRecipe == 0)
+ {
+ pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
+
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ instruction.mSpaceBefore = fileSize; // whole file
+ instruction.mBlocks = 0; // no blocks
+ instruction.mpStartBlock = 0; // no block
+ pblankRecipe->push_back(instruction);
+
+ pRecipe = pblankRecipe;
+ }
+
+ // Tell caller?
+ if(pModificationTime != 0)
+ {
+ *pModificationTime = modTime;
+ }
+
+ // Go through each instruction in the recipe and work out how many blocks
+ // it will add, and the max clear size of these blocks
+ int maxBlockClearSize = 0;
+ for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
+ {
+ if((*pRecipe)[inst].mSpaceBefore > 0)
+ {
+ // Calculate the number of blocks the space before requires
+ int64_t numBlocks;
+ int32_t blockSize, lastBlockSize;
+ CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
+ // Add to accumlated total
+ mTotalBlocks += numBlocks;
+ // Update maximum clear size
+ if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
+ if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
+ }
+
+ // Add number of blocks copied from the previous file
+ mTotalBlocks += (*pRecipe)[inst].mBlocks;
+
+ // Check for bad things
+ if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Run through blocks to get the max clear size
+ for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
+ {
+ if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
+ }
+ }
+
+ // Send data? (symlinks don't have any data in them)
+ mSendData = !attr.IsSymLink();
+
+ // If not data is being sent, then the max clear block size is zero
+ if(!mSendData)
+ {
+ maxBlockClearSize = 0;
+ }
+
+ // Header
+ file_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
+ hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0);
+ hdr.mContainerID = box_hton64(ContainerID);
+ hdr.mModificationTime = box_hton64(modTime);
+ // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
+ hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
+ hdr.mOptions = 0; // no options defined yet
+
+ // Write header to stream
+ mData.Write(&hdr, sizeof(hdr));
+
+ // Write filename to stream
+ rStoreFilename.WriteToStream(mData);
+
+ // Write attributes to stream
+ attr.WriteToStream(mData);
+
+ // Allocate some buffers for writing data
+ if(mSendData)
+ {
+ // Open the file
+ mpFile = new FileStream(Filename);
+
+ if (pLogger)
+ {
+ // Create logging stream
+ mpLogging = new ReadLoggingStream(*mpFile,
+ *pLogger);
+ }
+ else
+ {
+ // re-use FileStream instead
+ mpLogging = mpFile;
+ mpFile = NULL;
+ }
+
+ // Work out the largest possible block required for the encoded data
+ mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
+
+ // Then allocate two blocks of this size
+ mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
+ if(mpRawBuffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+#ifndef BOX_RELEASE_BUILD
+ // In debug builds, make sure that the reallocation code is exercised.
+ mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
+#else
+ mEncodedBuffer.Allocate(mAllocatedBufferSize);
+#endif
+ }
+ else
+ {
+ // Write an empty block index for the symlink
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ blkhdr.mOtherFileID = box_hton64(0); // not other file ID
+ blkhdr.mEntryIVBase = box_hton64(0);
+ blkhdr.mNumBlocks = box_hton64(0);
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ // Ready for reading
+ mData.SetForReading();
+
+ // Update stats
+ BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
+
+ // Finally, store the pointer to the recipe, when we know exceptions won't occur
+ mpRecipe = pRecipe;
+ }
+ catch(...)
+ {
+ // Clean up any blank recipe
+ if(pblankRecipe != 0)
+ {
+ delete pblankRecipe;
+ pblankRecipe = 0;
+ }
+ throw;
+ }
+
+ mpRunStatusProvider = pRunStatusProvider;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
+// Purpose: Calculates the sizes of blocks in a section of the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
+{
+ // How many blocks, and how big?
+ rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
+ do
+ {
+ rBlockSizeOut *= 2;
+
+ rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
+
+ } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
+
+ // Last block size
+ rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
+
+ // Avoid small blocks?
+ if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
+ && rNumBlocksOut > 1)
+ {
+ // Add the small bit of data to the last block
+ --rNumBlocksOut;
+ rLastBlockSizeOut += rBlockSizeOut;
+ }
+
+ // checks!
+ ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
+ //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Read(void *, int, int)
+// Purpose: As interface -- generates encoded file data on the fly from the raw file
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Check there's something to do.
+ if(mStatus == Status_Finished)
+ {
+ return 0;
+ }
+
+ if(mpRunStatusProvider && mpRunStatusProvider->StopRun())
+ {
+ THROW_EXCEPTION(BackupStoreException, SignalReceived);
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *buffer = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mStatus != Status_Finished)
+ {
+ if(mStatus == Status_Header || mStatus == Status_BlockListing)
+ {
+ // Header or block listing phase -- send from the buffered stream
+
+ // Send bytes from the data buffer
+ int b = mData.Read(buffer, bytesToRead, Timeout);
+ bytesToRead -= b;
+ buffer += b;
+
+ // Check to see if all the data has been used from this stream
+ if(!mData.StreamDataLeft())
+ {
+ // Yes, move on to next phase (or finish, if there's no file data)
+ if(!mSendData)
+ {
+ mStatus = Status_Finished;
+ }
+ else
+ {
+ // Reset the buffer so it can be used for the next phase
+ mData.Reset();
+
+ // Get buffer ready for index?
+ if(mStatus == Status_Header)
+ {
+ // Just finished doing the stream header, create the block index header
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ ASSERT(mpRecipe != 0);
+ blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID());
+ blkhdr.mNumBlocks = box_hton64(mTotalBlocks);
+
+ // Generate the IV base
+ Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
+ blkhdr.mEntryIVBase = box_hton64(mEntryIVBase);
+
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ ++mStatus;
+ }
+ }
+ }
+ else if(mStatus == Status_Blocks)
+ {
+ // Block sending phase
+
+ if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
+ {
+ // Next block!
+ ++mCurrentBlock;
+ ++mAbsoluteBlockNumber;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Output extra blocks for this instruction and move forward in file
+ if(mInstructionNumber >= 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ }
+
+ // Is there another instruction to go?
+ ++mInstructionNumber;
+
+ // Skip instructions which don't contain any data
+ while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
+ && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ ++mInstructionNumber;
+ }
+
+ if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
+ {
+ // End of blocks, go to next phase
+ ++mStatus;
+
+ // Set the data to reading so the index can be written
+ mData.SetForReading();
+ }
+ else
+ {
+ // Get ready for this instruction
+ SetForInstruction();
+ }
+ }
+
+ // Can't use 'else' here as SetForInstruction() will change this
+ if(mCurrentBlock < mNumBlocks)
+ {
+ EncodeCurrentBlock();
+ }
+ }
+
+ // Send data from the current block (if there's data to send)
+ if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
+ {
+ // How much data to put in the buffer?
+ int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead;
+
+ // Copy it in
+ ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
+
+ // Update variables
+ bytesToRead -= s;
+ buffer += s;
+ mPositionInCurrentBlock += s;
+ }
+ }
+ else
+ {
+ // Should never get here, as it'd be an invalid status
+ ASSERT(false);
+ }
+ }
+
+ // Add encoded size to stats
+ BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
+ mTotalBytesSent += (NBytes - bytesToRead);
+
+ // Return size of data to caller
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
+// Purpose: Private. Stores the blocks of the old file referenced in the current
+// instruction into the index and skips over the data in the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
+{
+ // Check something is necessary
+ if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
+ {
+ return;
+ }
+
+ // Index of the first block in old file (being diffed from)
+ int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
+
+ int64_t sizeToSkip = 0;
+
+ for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
+ {
+ // Update stats
+ BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+
+ // Store the entry
+ StoreBlockIndexEntry(0 - (firstIndex + b),
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);
+
+ // Increment the absolute block number -- kept encryption IV in sync
+ ++mAbsoluteBlockNumber;
+
+ // Add the size of this block to the size to skip
+ sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+ }
+
+ // Move forward in the stream
+ mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::SetForInstruction()
+// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SetForInstruction()
+{
+ // Calculate block sizes
+ CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
+
+ // Set variables
+ mCurrentBlock = 0;
+ mCurrentBlockEncodedSize = 0;
+ mPositionInCurrentBlock = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock()
+// Purpose: Private. Encodes the current block, and writes the block data to the index
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::EncodeCurrentBlock()
+{
+ // How big is the block, raw?
+ int blockRawSize = mBlockSize;
+ if(mCurrentBlock == (mNumBlocks - 1))
+ {
+ blockRawSize = mLastBlockSize;
+ }
+ ASSERT(blockRawSize < mAllocatedBufferSize);
+
+ // Check file open
+ if(mpLogging == 0)
+ {
+ // File should be open, but isn't. So logical error.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Read the data in
+ if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize,
+ 0 /* not interested in size if failure */))
+ {
+ // TODO: Do something more intelligent, and abort
+ // this upload because the file has changed.
+ THROW_EXCEPTION(BackupStoreException,
+ Temp_FileEncodeStreamDidntReadBuffer)
+ }
+
+ // Encode it
+ mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer,
+ blockRawSize, mEncodedBuffer);
+
+ //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
+
+ // Create block listing data -- generate checksums
+ RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
+ MD5Digest strongChecksum;
+ strongChecksum.Add(mpRawBuffer, blockRawSize);
+ strongChecksum.Finish();
+
+ // Add entry to the index
+ StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize,
+ weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
+
+ // Set vars to reading this block
+ mPositionInCurrentBlock = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
+// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
+{
+ // First, the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ entryEnc.mSize = htonl(ClearSize);
+ entryEnc.mWeakChecksum = htonl(WeakChecksum);
+ ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
+
+ // Then the clear section
+ file_BlockIndexEntry entry;
+ entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex));
+
+ // Then encrypt the encryted section
+ // Generate the IV from the block number
+ if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
+ {
+ THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
+ }
+ uint64_t iv = mEntryIVBase;
+ iv += mAbsoluteBlockNumber;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = box_hton64(iv);
+ sBlowfishEncryptBlockEntry.SetIV(&iv);
+
+ // Encode the data
+ int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
+ if(encodedSize != sizeof(entry.mEnEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Save to data block for sending at the end of the stream
+ mData.Write(&entry, sizeof(entry));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Write(const void *, int)
+// Purpose: As interface. Exceptions.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamDataLeft()
+// Purpose: As interface -- end of stream reached?
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamDataLeft()
+{
+ return (mStatus != Status_Finished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamClosed()
+// Purpose: As interface
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamClosed()
+{
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
+// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
+ int64_t NumBlocksInIndex, int64_t OtherFileID)
+ : mpBlockIndex(pBlockIndex),
+ mNumBlocksInIndex(NumBlocksInIndex),
+ mOtherFileID(OtherFileID)
+{
+ ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::~Recipe()
+// Purpose: Destructor
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::~Recipe()
+{
+ // Free the block index, if there is one
+ if(mpBlockIndex != 0)
+ {
+ ::free(mpBlockIndex);
+ }
+}
+
+
+
+
Copied: box/trunk/lib/backupstore/BackupStoreFileEncodeStream.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreFileEncodeStream.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFileEncodeStream.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFileEncodeStream.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,137 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.h
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEENCODESTREAM__H
+#define BACKUPSTOREFILEENCODESTREAM__H
+
+#include <vector>
+
+#include "IOStream.h"
+#include "BackupStoreFilename.h"
+#include "CollectInBufferStream.h"
+#include "MD5Digest.h"
+#include "BackupStoreFile.h"
+#include "ReadLoggingStream.h"
+#include "RunStatusProvider.h"
+
+namespace BackupStoreFileCreation
+{
+ // Diffing and creation of files share some implementation details.
+ typedef struct _BlocksAvailableEntry
+ {
+ struct _BlocksAvailableEntry *mpNextInHashList;
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+ } BlocksAvailableEntry;
+
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFileEncodeStream
+// Purpose: Encode a file into a stream
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+class BackupStoreFileEncodeStream : public IOStream
+{
+public:
+ BackupStoreFileEncodeStream();
+ ~BackupStoreFileEncodeStream();
+
+ typedef struct
+ {
+ int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go
+ int32_t mBlocks; // number of block to reuse, starting at this one
+ BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null
+ } RecipeInstruction;
+
+ class Recipe : public std::vector<RecipeInstruction>
+ {
+ // NOTE: This class is rather tied in with the implementation of diffing.
+ public:
+ Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex,
+ int64_t OtherFileID = 0);
+ ~Recipe();
+
+ int64_t GetOtherFileID() {return mOtherFileID;}
+ int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock)
+ {
+ return pBlock - mpBlockIndex;
+ }
+
+ private:
+ BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex;
+ int64_t mNumBlocksInIndex;
+ int64_t mOtherFileID;
+ };
+
+ void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename,
+ int64_t *pModificationTime,
+ ReadLoggingStream::Logger* pLogger = NULL,
+ RunStatusProvider* pRunStatusProvider = NULL);
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+ int64_t GetTotalBytesSent() { return mTotalBytesSent; }
+
+private:
+ enum
+ {
+ Status_Header = 0,
+ Status_Blocks = 1,
+ Status_BlockListing = 2,
+ Status_Finished = 3
+ };
+
+private:
+ void EncodeCurrentBlock();
+ void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut);
+ void SkipPreviousBlocksInInstruction();
+ void SetForInstruction();
+ void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum);
+
+private:
+ Recipe *mpRecipe;
+ IOStream *mpFile; // source file
+ CollectInBufferStream mData; // buffer for header and index entries
+ IOStream *mpLogging;
+ RunStatusProvider* mpRunStatusProvider;
+ int mStatus;
+ bool mSendData; // true if there's file data to send (ie not a symlink)
+ int64_t mTotalBlocks; // Total number of blocks in the file
+ int64_t mAbsoluteBlockNumber; // The absolute block number currently being output
+ // Instruction number
+ int64_t mInstructionNumber;
+ // All the below are within the current instruction
+ int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
+ int64_t mCurrentBlock;
+ int32_t mCurrentBlockEncodedSize;
+ int32_t mPositionInCurrentBlock; // for reading out
+ int32_t mBlockSize; // Basic block size of most of the blocks in the file
+ int32_t mLastBlockSize; // the size (unencoded) of the last block in the file
+ int64_t mTotalBytesSent;
+ // Buffers
+ uint8_t *mpRawBuffer; // buffer for raw data
+ BackupStoreFile::EncodingBuffer mEncodedBuffer;
+ // buffer for encoded data
+ int32_t mAllocatedBufferSize; // size of above two allocated blocks
+ uint64_t mEntryIVBase; // base for block entry IV
+};
+
+
+
+#endif // BACKUPSTOREFILEENCODESTREAM__H
+
Copied: box/trunk/lib/backupstore/BackupStoreFileRevDiff.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreFileRevDiff.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFileRevDiff.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFileRevDiff.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,258 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileRevDiff.cpp
+// Purpose: Reverse a patch, to build a new patch from new to old files
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t)
+// Purpose: Reverse a patch, to build a new patch from new to old files. Takes
+// two independent copies to the From file, for efficiency.
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent)
+{
+ // Read and copy the header from the from file to the out file -- beginnings of the patch
+ file_StreamFormat hdr;
+ if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Build an index of common blocks.
+ // For each block in the from file, we want to know it's index in the
+ // diff file. Allocate memory for this information.
+ int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks);
+ int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t));
+ if(pfromIndexInfo == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ // flag
+ bool isCompletelyDifferent = true;
+
+ try
+ {
+ // Initialise the index to be all 0, ie not filled in yet
+ for(int64_t i = 0; i < fromNumBlocks; ++i)
+ {
+ pfromIndexInfo[i] = 0;
+ }
+
+ // Within the from file, skip to the index
+ MoveStreamPositionToBlockIndex(rDiff);
+
+ // Read in header of index
+ file_BlockIndexHeader diffIdxHdr;
+ if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then read in each entry
+ int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks);
+ for(int64_t b = 0; b < diffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = box_ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file
+ }
+ else
+ {
+ // Block is in the original file, store which block it is in this file
+ int64_t fromIndex = 0 - blockEn;
+ if(fromIndex < 0 || fromIndex >= fromNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles)
+ }
+
+ // Store information about where it is in the new file
+ // NOTE: This is slight different to how it'll be stored in the final index.
+ pfromIndexInfo[fromIndex] = -1 - b;
+ }
+ }
+
+ // Open the index for the second copy of the from file
+ MoveStreamPositionToBlockIndex(rFrom2);
+
+ // Read in header of index
+ file_BlockIndexHeader fromIdxHdr;
+ if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || box_ntoh64(fromIdxHdr.mOtherFileID) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // So, we can now start building the data in the file
+ int64_t filePosition = rFrom.GetPosition();
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Get size
+ int64_t blockSize = box_hton64(e.mEncodedSize);
+ if(blockSize < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Copy this block?
+ if(pfromIndexInfo[b] == 0)
+ {
+ // Copy it, first move to file location
+ rFrom.Seek(filePosition, IOStream::SeekType_Absolute);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Copy the block
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ rOut.Write(buffer, blockSize);
+
+ // Store the size
+ pfromIndexInfo[b] = blockSize;
+ }
+ else
+ {
+ // Block isn't needed, so it's not completely different
+ isCompletelyDifferent = false;
+ }
+ filePosition += blockSize;
+ }
+
+ // Then write the index, modified header first
+ fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom));
+ rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr));
+
+ // Move to start of index entries
+ rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute);
+
+ // Then copy modified entries
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Modify...
+ int64_t s = pfromIndexInfo[b];
+ // Adjust to reflect real block index (remember 0 has a different meaning here)
+ if(s < 0) ++s;
+ // Insert
+ e.mEncodedSize = box_hton64(s);
+ // Write
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Free memory used (oh for finally {} blocks)
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+
+ // return completely different flag
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = isCompletelyDifferent;
+ }
+}
+
+
+
Copied: box/trunk/lib/backupstore/BackupStoreFileWire.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreFileWire.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFileWire.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFileWire.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,74 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileWire.h
+// Purpose: On the wire / disc formats for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEWIRE__H
+#define BACKUPSTOREFILEWIRE__H
+
+#include "MD5Digest.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int64_t mNumBlocks; // number of blocks contained in the file
+ int64_t mContainerID;
+ int64_t mModificationTime;
+ int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block
+ int32_t mOptions; // bitmask of options used
+ // Then a BackupStoreFilename
+ // Then a BackupClientFileAttributes
+} file_StreamFormat;
+
+typedef struct
+{
+ int32_t mMagicValue; // different magic value
+ int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index
+ uint64_t mEntryIVBase; // base value for block IV
+ int64_t mNumBlocks; // repeat of value in file header
+} file_BlockIndexHeader;
+
+typedef struct
+{
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+} file_BlockIndexEntryEnc;
+
+typedef struct
+{
+ union
+ {
+ int64_t mEncodedSize; // size encoded, if > 0
+ int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0
+ };
+ uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section
+} file_BlockIndexEntry;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// header for blocks of compressed data in files
+#define HEADER_CHUNK_IS_COMPRESSED 1 // bit
+#define HEADER_ENCODING_SHIFT 1 // shift value
+#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7
+#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7
+
+
+#endif // BACKUPSTOREFILEWIRE__H
+
Copied: box/trunk/lib/backupstore/BackupStoreFilename.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreFilename.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFilename.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFilename.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,281 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.cpp
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilename.h"
+#include "Protocol.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename()
+// Purpose: Default constructor -- creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy)
+ : mEncryptedName(rToCopy.mEncryptedName)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::~BackupStoreFilename()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::~BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::CheckValid(bool)
+// Purpose: Checks the encoded filename for validity
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const
+{
+ bool ok = true;
+
+ if(mEncryptedName.size() < 2)
+ {
+ // Isn't long enough to have a header
+ ok = false;
+ }
+ else
+ {
+ // Check size is consistent
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName);
+ if(dsize != mEncryptedName.size())
+ {
+ ok = false;
+ }
+
+ // And encoding is an accepted value
+ unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName);
+ if(encoding < Encoding_Min || encoding > Encoding_Max)
+ {
+ ok = false;
+ }
+ }
+
+ // Exception?
+ if(!ok && ExceptionIfInvalid)
+ {
+ THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename)
+ }
+
+ return ok;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromProtocol(Protocol &)
+// Purpose: Reads the filename from the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol)
+{
+ // Read the header
+ char hdr[2];
+ rProtocol.Read(hdr, 2);
+
+ // How big is it?
+ int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ std::string data;
+ rProtocol.Read(data, dsize - 2);
+
+ // assign to this string, storing the header and the extra data
+ mEncryptedName.assign(hdr, 2);
+ mEncryptedName.append(data.c_str(), data.size());
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToProtocol(Protocol &)
+// Purpose: Writes the filename to the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const
+{
+ CheckValid();
+
+ rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromStream(IOStream &)
+// Purpose: Reads the filename from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Read the header
+ char hdr[2];
+ if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // How big is it?
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Assume most filenames are small
+ char buf[256];
+ if(dsize < sizeof(buf))
+ {
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ buf[0] = hdr[0]; buf[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ mEncryptedName.assign(buf, dsize);
+ }
+ else
+ {
+ // Block of memory to hold it
+ MemoryBlockGuard<char*> dataB(dsize+2);
+ char *data = dataB;
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ data[0] = hdr[0]; data[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ mEncryptedName.assign(data, dsize);
+ }
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToStream(IOStream &)
+// Purpose: Writes the filename to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToStream(IOStream &rStream) const
+{
+ CheckValid();
+
+ rStream.Write(mEncryptedName.c_str(), mEncryptedName.size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::EncodedFilenameChanged()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::IsEncrypted()
+// Purpose: Returns true if the filename is stored using an encrypting encoding
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::IsEncrypted() const
+{
+ return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) !=
+ Encoding_Clear;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::SetAsClearFilename(const char *)
+// Purpose: Sets this object to be a valid filename, but with a
+// filename in the clear. Used on the server to create
+// filenames when there's no way of encrypting it.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::SetAsClearFilename(const char *Clear)
+{
+ // Make std::string from the clear name
+ std::string toEncode(Clear);
+
+ // Make an encoded string
+ char hdr[2];
+ BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear);
+ std::string encoded(hdr, 2);
+ encoded += toEncode;
+ ASSERT(encoded.size() == toEncode.size() + 2);
+
+ // Store the encoded string
+ mEncryptedName.assign(encoded);
+
+ // Stuff which must be done
+ EncodedFilenameChanged();
+ CheckValid(false);
+}
+
+
+
Copied: box/trunk/lib/backupstore/BackupStoreFilename.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreFilename.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFilename.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFilename.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,107 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.h
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAME__H
+#define BACKUPSTOREFILENAME__H
+
+#include <string>
+
+class Protocol;
+class IOStream;
+
+// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+// don't define this -- the problem of memory usage still appears without this.
+// It's just that this class really showed up the problem. Instead, malloc allocation
+// is globally defined in BoxPlatform.h, for troublesome libraries.
+
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ // Use a malloc_allocated string, because the STL default allocators really screw up with
+ // memory allocation, particularly with this class.
+ // Makes a few things a bit messy and inefficient with conversions.
+ // Given up using this, and use global malloc allocation instead, but thought it
+ // worth leaving this code in just in case it's useful for the future.
+ typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base;
+ // If this is changed, change GetClearFilename() back to returning a reference.
+#else
+ typedef std::string BackupStoreFilename_base;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilename
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilename /* : public BackupStoreFilename_base */
+{
+private:
+ std::string mEncryptedName;
+
+public:
+ BackupStoreFilename();
+ BackupStoreFilename(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilename();
+
+ bool CheckValid(bool ExceptionIfInvalid = true) const;
+
+ void ReadFromProtocol(Protocol &rProtocol);
+ void WriteToProtocol(Protocol &rProtocol) const;
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ void SetAsClearFilename(const char *Clear);
+
+ // Check that it's encrypted
+ bool IsEncrypted() const;
+
+ // These enumerated types belong in the base class so
+ // the CheckValid() function can make sure that the encoding
+ // is a valid encoding
+ enum
+ {
+ Encoding_Min = 1,
+ Encoding_Clear = 1,
+ Encoding_Blowfish = 2,
+ Encoding_Max = 2
+ };
+
+ const std::string& GetEncodedFilename() const
+ {
+ return mEncryptedName;
+ }
+
+ bool operator==(const BackupStoreFilename& rOther) const
+ {
+ return mEncryptedName == rOther.mEncryptedName;
+ }
+
+ bool operator!=(const BackupStoreFilename& rOther) const
+ {
+ return mEncryptedName != rOther.mEncryptedName;
+ }
+
+protected:
+ virtual void EncodedFilenameChanged();
+ void SetEncodedFilename(const std::string &rEncoded)
+ {
+ mEncryptedName = rEncoded;
+ }
+};
+
+// On the wire utilities for class and derived class
+#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
+#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3)
+
+#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;}
+
+#endif // BACKUPSTOREFILENAME__H
+
Copied: box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp (from rev 2943, box/trunk/lib/backupclient/BackupStoreFilenameClear.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,335 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.cpp
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "Guards.h"
+#include "Logging.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide private variables from the rest of the world
+namespace
+{
+ int sEncodeMethod = BackupStoreFilename::Encoding_Clear;
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear()
+// Purpose: Default constructor, creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &)
+// Purpose: Creates a filename, encoding from the given string
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode)
+{
+ SetClearFilename(rToEncode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy)
+ : BackupStoreFilename(rToCopy),
+ mClearFilename(rToCopy.mClearFilename)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+// Purpose: Copy from base class
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename(rToCopy)
+{
+ // Will get a clear filename when it's required
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::~BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::GetClearFilename()
+// Purpose: Get the unencoded filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+const std::string BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ // When modifying, remember to change back to reference return if at all possible
+ // -- returns an object rather than a reference to allow easy use with other code.
+ return std::string(mClearFilename.c_str(), mClearFilename.size());
+}
+#else
+const std::string &BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ return mClearFilename;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &)
+// Purpose: Encode and make available the clear filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode)
+{
+ // Only allow Blowfish encodings
+ if(sEncodeMethod != Encoding_Blowfish)
+ {
+ THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup)
+ }
+
+ // Make an encoded string with blowfish encryption
+ EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish);
+
+ // Store the clear filename
+ mClearFilename.assign(rToEncode.c_str(), rToEncode.size());
+
+ // Make sure we did the right thing
+ if(!CheckValid(false))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::MakeClearAvailable()
+// Purpose: Private. Make sure the clear filename is available
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::MakeClearAvailable() const
+{
+ if(!mClearFilename.empty())
+ return; // nothing to do
+
+ // Check valid
+ CheckValid();
+
+ // Decode the header
+ int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename());
+ int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename());
+
+ // Decode based on encoding given in the header
+ switch(encoding)
+ {
+ case Encoding_Clear:
+ BOX_TRACE("**** BackupStoreFilename encoded with "
+ "Clear encoding ****");
+ mClearFilename.assign(GetEncodedFilename().c_str() + 2,
+ size - 2);
+ break;
+
+ case Encoding_Blowfish:
+ DecryptEncoded(sBlowfishDecrypt);
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding)
+ break;
+ }
+}
+
+
+// Buffer for encoding and decoding -- do this all in one single buffer to
+// avoid lots of string allocation, which stuffs up memory usage.
+// These static memory vars are, of course, not thread safe, but we don't use threads.
+static int sEncDecBufferSize = 0;
+static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0;
+
+static void EnsureEncDecBufferSize(int BufSize)
+{
+ if(spEncDecBuffer == 0)
+ {
+#ifndef WIN32
+ BOX_TRACE("Allocating filename encoding/decoding buffer "
+ "with size " << BufSize);
+#endif
+ spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize);
+ MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer);
+ MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer);
+ sEncDecBufferSize = BufSize;
+ }
+ else
+ {
+ if(sEncDecBufferSize < BufSize)
+ {
+ BOX_TRACE("Reallocating filename encoding/decoding "
+ "buffer from " << sEncDecBufferSize <<
+ " to " << BufSize);
+ spEncDecBuffer->Resize(BufSize);
+ sEncDecBufferSize = BufSize;
+ MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int)
+// Purpose: Private. Assigns the encoded filename string, encrypting.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding)
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ EnsureEncDecBufferSize(maxOutSize);
+
+ // Pointer to buffer
+ uint8_t *buffer = *spEncDecBuffer;
+
+ // Encode -- do entire block in one go
+ int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size());
+ // and add in header size
+ encSize += 2;
+
+ // Adjust header
+ BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding);
+
+ // Store the encoded string
+ SetEncodedFilename(std::string((char*)buffer, encSize));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &)
+// Purpose: Decrypt the encoded filename using the cipher context
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const
+{
+ const std::string& rEncoded = GetEncodedFilename();
+
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ EnsureEncDecBufferSize(maxOutSize);
+
+ // Pointer to buffer
+ uint8_t *buffer = *spEncDecBuffer;
+
+ // Decrypt
+ const char *str = rEncoded.c_str() + 2;
+ int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2);
+
+ // Assign to this
+ mClearFilename.assign((char*)buffer, sizeOut);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncodedFilenameChanged()
+{
+ BackupStoreFilename::EncodedFilenameChanged();
+
+ // Delete stored filename in clear
+ mClearFilename.erase();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int)
+// Purpose: Set the key used for Blowfish encryption of filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength)
+{
+ // Initialisation vector not used. Can't use a different vector for each filename as
+ // that would stop comparisions on the server working.
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength);
+ sBlowfishEncrypt.SetIV(pIV);
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength);
+ sBlowfishDecrypt.SetIV(pIV);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetEncodingMethod(int)
+// Purpose: Set the encoding method used for filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetEncodingMethod(int Method)
+{
+ sEncodeMethod = Method;
+}
+
+
+
Copied: box/trunk/lib/backupstore/BackupStoreFilenameClear.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreFilenameClear.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreFilenameClear.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreFilenameClear.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.h
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAMECLEAR__H
+#define BACKUPSTOREFILENAMECLEAR__H
+
+#include "BackupStoreFilename.h"
+
+class CipherContext;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilenameClear
+// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilenameClear : public BackupStoreFilename
+{
+public:
+ BackupStoreFilenameClear();
+ BackupStoreFilenameClear(const std::string &rToEncode);
+ BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy);
+ BackupStoreFilenameClear(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilenameClear();
+
+ // Because we need to use a different allocator for this class to avoid
+ // nasty things happening, can't return this as a reference. Which is a
+ // pity. But probably not too bad.
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ const std::string GetClearFilename() const;
+#else
+ const std::string &GetClearFilename() const;
+#endif
+ void SetClearFilename(const std::string &rToEncode);
+
+ // Setup for encryption of filenames
+ static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength);
+ static void SetEncodingMethod(int Method);
+
+protected:
+ void MakeClearAvailable() const;
+ virtual void EncodedFilenameChanged();
+ void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding);
+ void DecryptEncoded(CipherContext &rCipherContext) const;
+
+private:
+ mutable BackupStoreFilename_base mClearFilename;
+};
+
+#endif // BACKUPSTOREFILENAMECLEAR__H
+
+
Copied: box/trunk/lib/backupstore/BackupStoreObjectMagic.h (from rev 2943, box/trunk/lib/backupclient/BackupStoreObjectMagic.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreObjectMagic.h (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreObjectMagic.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectMagic.h
+// Purpose: Magic values for the start of objects in the backup store
+// Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREOBJECTMAGIC__H
+#define BACKUPSTOREOBJECTMAGIC__H
+
+// Each of these values is the first 4 bytes of the object file.
+// Remember to swap from network to host byte order.
+
+// Magic value for file streams
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45
+
+// Magic for the block index at the file stream -- used to
+// ensure streams are reordered as expected
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B
+
+// Magic value for directory streams
+#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F
+
+#endif // BACKUPSTOREOBJECTMAGIC__H
+
Copied: box/trunk/lib/backupstore/Makefile.extra (from rev 2943, box/trunk/lib/backupclient/Makefile.extra)
===================================================================
--- box/trunk/lib/backupstore/Makefile.extra (rev 0)
+++ box/trunk/lib/backupstore/Makefile.extra 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,20 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_CLI = $(MAKEPROTOCOL) Client backupprotocol.txt
+GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) backupprotocol.txt
+ $(_PERL) $(GEN_CMD_CLI)
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h: $(MAKEPROTOCOL) backupprotocol.txt
+ $(_PERL) $(GEN_CMD_SRV)
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt
+ $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt
+
Copied: box/trunk/lib/backupstore/RunStatusProvider.h (from rev 2943, box/trunk/lib/backupclient/RunStatusProvider.h)
===================================================================
--- box/trunk/lib/backupstore/RunStatusProvider.h (rev 0)
+++ box/trunk/lib/backupstore/RunStatusProvider.h 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,29 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: RunStatusProvider.h
+// Purpose: Declares the RunStatusProvider interface.
+// Created: 2008/08/14
+//
+// --------------------------------------------------------------------------
+
+#ifndef RUNSTATUSPROVIDER__H
+#define RUNSTATUSPROVIDER__H
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: RunStatusProvider
+// Purpose: Provides a StopRun() method which returns true if
+// the current backup should be halted.
+// Created: 2005/11/15
+//
+// --------------------------------------------------------------------------
+class RunStatusProvider
+{
+ public:
+ virtual ~RunStatusProvider() { }
+ virtual bool StopRun() = 0;
+};
+
+#endif // RUNSTATUSPROVIDER__H
Copied: box/trunk/lib/backupstore/backupprotocol.txt (from rev 2943, box/trunk/bin/bbstored/backupprotocol.txt)
===================================================================
--- box/trunk/lib/backupstore/backupprotocol.txt (rev 0)
+++ box/trunk/lib/backupstore/backupprotocol.txt 2011-04-26 18:44:26 UTC (rev 2945)
@@ -0,0 +1,235 @@
+#
+# backup protocol definition
+#
+
+Name Backup
+IdentString Box-Backup:v=C
+ServerContextClass BackupStoreContext BackupStoreContext.h
+
+ClientType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h
+ServerType Filename BackupStoreFilename BackupStoreFilename.h
+
+ImplementLog Server syslog
+ImplementLog Client syslog
+ImplementLog Client file
+
+LogTypeToText Client Filename \"%s\" VAR.GetClearFilename().c_str()
+
+BEGIN_OBJECTS
+
+# -------------------------------------------------------------------------------------
+# Session commands
+# -------------------------------------------------------------------------------------
+
+Error 0 IsError(Type,SubType) Reply
+ int32 Type
+ int32 SubType
+ CONSTANT ErrorType 1000
+ CONSTANT Err_WrongVersion 1
+ CONSTANT Err_NotInRightProtocolPhase 2
+ CONSTANT Err_BadLogin 3
+ CONSTANT Err_CannotLockStoreForWriting 4
+ CONSTANT Err_SessionReadOnly 5
+ CONSTANT Err_FileDoesNotVerify 6
+ CONSTANT Err_DoesNotExist 7
+ CONSTANT Err_DirectoryAlreadyExists 8
+ CONSTANT Err_CannotDeleteRoot 9
+ CONSTANT Err_TargetNameExists 10
+ CONSTANT Err_StorageLimitExceeded 11
+ CONSTANT Err_DiffFromFileDoesNotExist 12
+ CONSTANT Err_DoesNotExistInDirectory 13
+ CONSTANT Err_PatchConsistencyError 14
+ CONSTANT Err_MultiplyReferencedObject 15
+
+Version 1 Command(Version) Reply
+ int32 Version
+
+
+Login 2 Command(LoginConfirmed)
+ int32 ClientID
+ int32 Flags
+ CONSTANT Flags_ReadOnly 1
+
+
+LoginConfirmed 3 Reply
+ int64 ClientStoreMarker
+ int64 BlocksUsed
+ int64 BlocksSoftLimit
+ int64 BlocksHardLimit
+
+
+Finished 4 Command(Finished) Reply EndsConversation
+
+
+# generic success object
+Success 5 Reply
+ int64 ObjectID
+
+
+SetClientStoreMarker 6 Command(Success)
+ int64 ClientStoreMarker
+
+
+# -------------------------------------------------------------------------------------
+# Generic object commands
+# -------------------------------------------------------------------------------------
+
+GetObject 10 Command(Success)
+ int64 ObjectID
+ CONSTANT NoObject 0
+ # reply has stream following, if ObjectID != NoObject
+
+
+MoveObject 11 Command(Success)
+ int64 ObjectID
+ int64 MoveFromDirectory
+ int64 MoveToDirectory
+ int32 Flags
+ Filename NewFilename
+
+ CONSTANT Flags_MoveAllWithSameName 1
+ CONSTANT Flags_AllowMoveOverDeletedObject 2
+
+# consider this an object command as, although it deals with directory entries,
+# it's not specific to either a file or a directory
+
+
+GetObjectName 12 Command(ObjectName)
+ int64 ObjectID
+ int64 ContainingDirectoryID
+ CONSTANT ObjectID_DirectoryOnly 0
+
+ # set ObjectID to ObjectID_DirectoryOnly to only get info on the directory
+
+
+ObjectName 13 Reply
+ int32 NumNameElements
+ int64 ModificationTime
+ int64 AttributesHash
+ int16 Flags
+ # NumNameElements is zero if the object doesn't exist
+ CONSTANT NumNameElements_ObjectDoesntExist 0
+ # a stream of Filename objects follows, if and only if NumNameElements > 0
+
+
+# -------------------------------------------------------------------------------------
+# Directory commands
+# -------------------------------------------------------------------------------------
+
+CreateDirectory 20 Command(Success) StreamWithCommand
+ int64 ContainingDirectoryID
+ int64 AttributesModTime
+ Filename DirectoryName
+ # stream following containing attributes
+
+
+ListDirectory 21 Command(Success)
+ int64 ObjectID
+ int16 FlagsMustBeSet
+ int16 FlagsNotToBeSet
+ bool SendAttributes
+ # make sure these flags are synced with those in BackupStoreDirectory
+ CONSTANT Flags_INCLUDE_EVERYTHING -1
+ CONSTANT Flags_EXCLUDE_NOTHING 0
+ CONSTANT Flags_EXCLUDE_EVERYTHING 15
+ CONSTANT Flags_File 1
+ CONSTANT Flags_Dir 2
+ CONSTANT Flags_Deleted 4
+ CONSTANT Flags_OldVersion 8
+ # make sure this is the same as in BackupStoreConstants.h
+ CONSTANT RootDirectory 1
+
+ # reply has stream following Success object, containing a stored BackupStoreDirectory
+
+
+ChangeDirAttributes 22 Command(Success) StreamWithCommand
+ int64 ObjectID
+ int64 AttributesModTime
+ # stream following containing attributes
+
+
+DeleteDirectory 23 Command(Success)
+ int64 ObjectID
+
+UndeleteDirectory 24 Command(Success)
+ int64 ObjectID
+ # may not have exactly the desired effect if files within in have been deleted before the directory was deleted.
+
+
+# -------------------------------------------------------------------------------------
+# File commands
+# -------------------------------------------------------------------------------------
+
+StoreFile 30 Command(Success) StreamWithCommand
+ int64 DirectoryObjectID
+ int64 ModificationTime
+ int64 AttributesHash
+ int64 DiffFromFileID # 0 if the file is not a diff
+ Filename Filename
+ # then send a stream containing the encoded file
+
+
+GetFile 31 Command(Success)
+ int64 InDirectory
+ int64 ObjectID
+ # error returned if not a file, or does not exist
+ # reply has stream following, containing an encoded file IN STREAM ORDER
+ # (use GetObject to get it in file order)
+
+
+SetReplacementFileAttributes 32 Command(Success) StreamWithCommand
+ int64 InDirectory
+ int64 AttributesHash
+ Filename Filename
+ # stream follows containing attributes
+
+
+DeleteFile 33 Command(Success)
+ int64 InDirectory
+ Filename Filename
+ # will return 0 if the object couldn't be found in the specified directory
+
+
+GetBlockIndexByID 34 Command(Success)
+ int64 ObjectID
+
+ # stream of the block index follows the reply
+ # returns an error if the object didn't exist
+
+
+GetBlockIndexByName 35 Command(Success)
+ int64 InDirectory
+ Filename Filename
+
+ # Success object contains the found ID -- or 0 if the entry wasn't found in the directory
+ # stream of the block index follows the reply if found ID != 0
+
+
+UndeleteFile 36 Command(Success)
+ int64 InDirectory
+ int64 ObjectID
+ # will return 0 if the object couldn't be found in the specified directory
+
+
+# -------------------------------------------------------------------------------------
+# Information commands
+# -------------------------------------------------------------------------------------
+
+GetAccountUsage 40 Command(AccountUsage)
+ # no data members
+
+AccountUsage 41 Reply
+ int64 BlocksUsed
+ int64 BlocksInOldFiles
+ int64 BlocksInDeletedFiles
+ int64 BlocksInDirectories
+ int64 BlocksSoftLimit
+ int64 BlocksHardLimit
+ int32 BlockSize
+
+GetIsAlive 42 Command(IsAlive)
+ # no data members
+
+IsAlive 43 Reply
+ # no data members
+
Modified: box/trunk/modules.txt
===================================================================
--- box/trunk/modules.txt 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/modules.txt 2011-04-26 18:44:26 UTC (rev 2945)
@@ -25,8 +25,8 @@
# Backup system
-lib/backupclient lib/server lib/crypto lib/compress
-lib/backupstore lib/server lib/raidfile lib/backupclient
+lib/backupstore lib/server lib/raidfile lib/crypto lib/compress
+lib/backupclient lib/backupstore
bin/bbackupobjdump lib/backupclient lib/backupstore
bin/bbstored lib/raidfile lib/server lib/backupstore lib/backupclient
Modified: box/trunk/test/bbackupd/Makefile.extra
===================================================================
--- box/trunk/test/bbackupd/Makefile.extra 2011-04-23 10:19:19 UTC (rev 2944)
+++ box/trunk/test/bbackupd/Makefile.extra 2011-04-26 18:44:26 UTC (rev 2945)
@@ -9,6 +9,6 @@
../../bin/bbstored/BackupStoreContext.o \
../../bin/bbstored/BBStoreDHousekeeping.o \
../../bin/bbstored/HousekeepStoreAccount.o \
- ../../bin/bbstored/autogen_BackupProtocolServer.o \
+ ../../lib/backupstore/autogen_BackupProtocolServer.o \
../../bin/bbstored/BackupCommands.o \
../../bin/bbstored/BackupStoreDaemon.o
More information about the Boxbackup-commit
mailing list