From subversion at boxbackup.org Thu Apr 12 00:00:01 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 12 Apr 2012 00:00:01 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3089 - box/trunk/bin/bbackupd Message-ID: <20120411230001.E6D801A32B6@www.boxbackup.org> Author: chris Date: 2012-04-12 00:00:00 +0100 (Thu, 12 Apr 2012) New Revision: 3089 Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h Log: Improve logging when decrypting a filename fails during UpdateItems(). Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2012-03-16 00:41:38 UTC (rev 3088) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2012-04-11 23:00:00 UTC (rev 3089) @@ -731,7 +731,34 @@ } } +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath) +{ + BackupStoreFilenameClear fn(en->GetName()); + return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); +} +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreFilenameClear fn, int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath) +{ + std::string filenameClear; + try + { + filenameClear = fn.GetClearFilename(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to decrypt filename for object " << + BOX_FORMAT_OBJECTID(filenameObjectID) << " in " + "directory " << BOX_FORMAT_OBJECTID(mObjectID) << + " (" << rRemoteDirectoryPath << ")"); + throw; + } + return filenameClear; +} + // -------------------------------------------------------------------------- // // Function @@ -770,7 +797,9 @@ BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { - decryptedEntries[BackupStoreFilenameClear(en->GetName()).GetClearFilename()] = en; + std::string filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; } } @@ -1301,8 +1330,11 @@ // Get rid of it. BackupProtocolClient &connection(rContext.GetConnection()); connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + + std::string filenameClear = DecryptFilename(en, + rRemotePath); rNotifier.NotifyFileDeleted(en->GetObjectID(), - storeFilename.GetClearFilename()); + filenameClear); // Nothing found en = 0; @@ -1445,14 +1477,19 @@ // Create a new directory std::auto_ptr dirCreate( connection.QueryCreateDirectory( - mObjectID, attrModTime, storeFilename, - attrStream)); + mObjectID, attrModTime, + storeFilename, attrStream)); subDirObjectID = dirCreate->GetObjectID(); // Flag as having done this for optimisation later haveJustCreatedDirOnServer = true; - rNotifier.NotifyDirectoryCreated(subDirObjectID, - storeFilename.GetClearFilename(), + + std::string filenameClear = + DecryptFilename(storeFilename, + subDirObjectID, + rRemotePath); + rNotifier.NotifyDirectoryCreated( + subDirObjectID, filenameClear, nonVssDirPath); } } @@ -1514,9 +1551,11 @@ // aren't actually deleted, as the whole state will be reset anyway. BackupClientDeleteList &rdel(rContext.GetDeleteList()); - BackupStoreFilenameClear clear(en->GetName()); + std::string filenameClear = DecryptFilename(en, + rRemotePath); + std::string localName = MakeFullPath(rLocalPath, - clear.GetClearFilename()); + filenameClear); std::string nonVssLocalName = ConvertVssPathToRealPath(localName, rBackupLocation); @@ -1536,7 +1575,8 @@ // If there's a directory record for it in // the sub directory map, delete it now BackupStoreFilenameClear dirname(en->GetName()); - std::map::iterator e(mSubDirectories.find(dirname.GetClearFilename())); + std::map::iterator + e(mSubDirectories.find(filenameClear)); if(e != mSubDirectories.end()) { // Carefully delete the entry from the map Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h 2012-03-16 00:41:38 UTC (rev 3088) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h 2012-04-11 23:00:00 UTC (rev 3089) @@ -168,6 +168,11 @@ BackupStoreDirectory* pDirOnStore, BackupStoreDirectory::Entry* pEntry, const std::string &rFilename); + std::string DecryptFilename(BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath); + std::string DecryptFilename(BackupStoreFilenameClear fn, + int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath); int64_t mObjectID; std::string mSubDirName; From subversion at boxbackup.org Thu Apr 12 00:00:54 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 12 Apr 2012 00:00:54 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3090 - box/trunk/bin/bbackupd Message-ID: <20120411230054.B90311A33B0@www.boxbackup.org> Author: chris Date: 2012-04-12 00:00:54 +0100 (Thu, 12 Apr 2012) New Revision: 3090 Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp box/trunk/bin/bbackupd/BackupClientContext.h Log: Fix a memory leak when TcpNice is disabled. Modified: box/trunk/bin/bbackupd/BackupClientContext.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-04-11 23:00:00 UTC (rev 3089) +++ box/trunk/bin/bbackupd/BackupClientContext.cpp 2012-04-11 23:00:54 UTC (rev 3090) @@ -115,9 +115,10 @@ // there shouldn't be a connection open ASSERT(mapSocket.get() == 0); + // Defensive. Must close connection before releasing any old socket. + mapConnection.reset(); + mapSocket.reset(new SocketStreamTLS); - SocketStreamTLS *pSocket = new SocketStreamTLS; - try { // Defensive. @@ -128,20 +129,21 @@ mHostname << "'..."); // Connect! - pSocket->Open(mrTLSContext, Socket::TypeINET, mHostname.c_str(), mPort); + ((SocketStreamTLS *)(mapSocket.get()))->Open(mrTLSContext, + Socket::TypeINET, mHostname, mPort); if(mTcpNiceMode) { - mapNice.reset(new NiceSocketStream(std::auto_ptr(pSocket))); + // Pass control of mapSocket to NiceSocketStream, + // which will take care of destroying it for us. + mapNice.reset(new NiceSocketStream(mapSocket)); mapConnection.reset(new BackupProtocolClient(*mapNice)); } else { - mapConnection.reset(new BackupProtocolClient(*pSocket)); + mapConnection.reset(new BackupProtocolClient(*mapSocket)); } - pSocket = NULL; - // Set logging option mapConnection->SetLogToSysLog(mExtendedLogging); @@ -189,8 +191,8 @@ try { mapConnection->QueryFinished(); + mapNice.reset(); mapSocket.reset(); - mapNice.reset(); } catch(...) { @@ -219,8 +221,8 @@ { // Clean up. mapConnection.reset(); + mapNice.reset(); mapSocket.reset(); - mapNice.reset(); throw; } @@ -269,8 +271,8 @@ try { // Be nice about closing the socket + mapNice.reset(); mapSocket.reset(); - mapNice.reset(); } catch(...) { Modified: box/trunk/bin/bbackupd/BackupClientContext.h =================================================================== --- box/trunk/bin/bbackupd/BackupClientContext.h 2012-04-11 23:00:00 UTC (rev 3089) +++ box/trunk/bin/bbackupd/BackupClientContext.h 2012-04-11 23:00:54 UTC (rev 3090) @@ -224,7 +224,7 @@ std::string mHostname; int mPort; uint32_t mAccountNumber; - std::auto_ptr mapSocket; + std::auto_ptr mapSocket; std::auto_ptr mapNice; std::auto_ptr mapConnection; bool mExtendedLogging; From subversion at boxbackup.org Sun Apr 15 14:09:58 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 15 Apr 2012 14:09:58 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3091 - box/trunk/bin/bbackupd Message-ID: <20120415130958.D491E1A3763@www.boxbackup.org> Author: chris Date: 2012-04-15 14:09:57 +0100 (Sun, 15 Apr 2012) New Revision: 3091 Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp Log: Use C++ string instead of converting to C string and back. Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.cpp 2012-04-11 23:00:54 UTC (rev 3090) +++ box/trunk/bin/bbackupd/BackupDaemon.cpp 2012-04-15 13:09:57 UTC (rev 3091) @@ -513,7 +513,7 @@ keyFile.c_str(), caFile.c_str()); // Set up the keys for various things - BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); + BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); } // -------------------------------------------------------------------------- From subversion at boxbackup.org Sun Apr 15 14:11:59 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 15 Apr 2012 14:11:59 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3092 - box/trunk/bin/bbackupd Message-ID: <20120415131159.B84611A3784@www.boxbackup.org> Author: chris Date: 2012-04-15 14:11:59 +0100 (Sun, 15 Apr 2012) New Revision: 3092 Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp Log: Treat corrupt filenames (not decryptable) as not existing, so that the client will flag them for deletion, and the store will eventually prune them. We could probably recover better by flagging them for immediate deletion (Remove_ASAP) but this is a better-tested code path. Remove unused variable hasMultipleHardLinks. Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2012-04-15 13:09:57 UTC (rev 3091) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2012-04-15 13:11:59 UTC (rev 3092) @@ -18,6 +18,7 @@ #include #include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" #include "autogen_ClientException.h" #include "Archive.h" #include "BackupClientContext.h" @@ -797,9 +798,19 @@ BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { - std::string filenameClear = DecryptFilename(en, - rRemotePath); - decryptedEntries[filenameClear] = en; + std::string filenameClear; + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "pretending that the file doesn't " + "exist"); + } } } @@ -820,7 +831,6 @@ uint64_t attributesHash = 0; int64_t fileSize = 0; InodeRefType inodeNum = 0; - bool hasMultipleHardLinks = true; // BLOCK { // Stat the file @@ -842,7 +852,6 @@ modTime = FileModificationTime(st); fileSize = st.st_size; inodeNum = st.st_ino; - hasMultipleHardLinks = (st.st_nlink > 1); attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); } @@ -861,7 +870,7 @@ } // Check that the entry which might have been found is in fact a file - if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0)) + if((en != 0) && !(en->IsFile())) { // Directory exists in the place of this file -- sort it out RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, @@ -1324,7 +1333,7 @@ } // Check that the entry which might have been found is in fact a directory - if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0)) + if((en != 0) && !(en->IsDir())) { // Entry exists, but is not a directory. Bad. // Get rid of it. @@ -1550,9 +1559,21 @@ // If there's an error during the process, it doesn't matter if things // aren't actually deleted, as the whole state will be reset anyway. BackupClientDeleteList &rdel(rContext.GetDeleteList()); + std::string filenameClear; + bool isCorruptFilename = false; - std::string filenameClear = DecryptFilename(en, - rRemotePath); + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "scheduling that file for deletion"); + filenameClear = ""; + isCorruptFilename = true; + } std::string localName = MakeFullPath(rLocalPath, filenameClear); @@ -1577,7 +1598,7 @@ BackupStoreFilenameClear dirname(en->GetName()); std::map::iterator e(mSubDirectories.find(filenameClear)); - if(e != mSubDirectories.end()) + if(e != mSubDirectories.end() && !isCorruptFilename) { // Carefully delete the entry from the map BackupClientDirectoryRecord *rec = e->second; From subversion at boxbackup.org Sat Apr 28 19:07:43 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:07:43 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3093 - box/trunk/infrastructure/m4 Message-ID: <20120428180743.24FE11A3605@www.boxbackup.org> Author: chris Date: 2012-04-28 19:07:42 +0100 (Sat, 28 Apr 2012) New Revision: 3093 Modified: box/trunk/infrastructure/m4/boxbackup_tests.m4 Log: Move libdl tests before boxbackup tests in configure.ac to fix OpenSSL link errors. Modified: box/trunk/infrastructure/m4/boxbackup_tests.m4 =================================================================== --- box/trunk/infrastructure/m4/boxbackup_tests.m4 2012-04-15 13:11:59 UTC (rev 3092) +++ box/trunk/infrastructure/m4/boxbackup_tests.m4 2012-04-28 18:07:42 UTC (rev 3093) @@ -97,6 +97,10 @@ AX_SPLIT_VERSION([BDB_VERSION], [$BDB_VERSION]) ]) +# need to find libdl before trying to link openssl, apparently +AC_SEARCH_LIBS([dlsym], ["dl"]) +AC_CHECK_FUNCS([dlsym dladdr]) + ## Check for Open SSL, use old versions only if explicitly requested AC_SEARCH_LIBS([gethostbyname], [nsl socket resolv]) AC_SEARCH_LIBS([shutdown], [nsl socket resolv]) @@ -167,9 +171,6 @@ have_regex_support=no fi -AC_SEARCH_LIBS([dlsym], ["dl"]) -AC_CHECK_FUNCS([dlsym dladdr]) - ### Checks for typedefs, structures, and compiler characteristics. AC_CHECK_TYPES([u_int8_t, u_int16_t, u_int32_t, u_int64_t]) From subversion at boxbackup.org Sat Apr 28 19:08:11 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:08:11 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3094 - box/trunk/infrastructure/m4 Message-ID: <20120428180811.B50B51A3624@www.boxbackup.org> Author: chris Date: 2012-04-28 19:08:11 +0100 (Sat, 28 Apr 2012) New Revision: 3094 Modified: box/trunk/infrastructure/m4/ax_check_ssl.m4 Log: Check for -lcrypto before -lssl, to ensure correct order in LIBS and successful link. Modified: box/trunk/infrastructure/m4/ax_check_ssl.m4 =================================================================== --- box/trunk/infrastructure/m4/ax_check_ssl.m4 2012-04-28 18:07:42 UTC (rev 3093) +++ box/trunk/infrastructure/m4/ax_check_ssl.m4 2012-04-28 18:08:11 UTC (rev 3094) @@ -27,7 +27,8 @@ ax_check_ssl_found=yes AC_CHECK_HEADERS([openssl/ssl.h],, [ax_check_ssl_found=no]) - AC_CHECK_LIB([ssl], [SSL_read],, [ax_check_ssl_found=no], [-lcrypto]) + AC_CHECK_LIB([crypto], [HMAC_CTX_init]) + AC_CHECK_LIB([ssl], [SSL_read],, [ax_check_ssl_found=no]) if test "x$ax_check_ssl_found" = "xyes"; then AC_DEFINE([HAVE_SSL], 1, [Define to 1 if SSL is available]) From subversion at boxbackup.org Sat Apr 28 19:08:28 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:08:28 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3095 - box/trunk Message-ID: <20120428180828.99C0E1A364C@www.boxbackup.org> Author: chris Date: 2012-04-28 19:08:28 +0100 (Sat, 28 Apr 2012) New Revision: 3095 Modified: box/trunk/modules.txt Log: Server module will shortly depend on crypto. Modified: box/trunk/modules.txt =================================================================== --- box/trunk/modules.txt 2012-04-28 18:08:11 UTC (rev 3094) +++ box/trunk/modules.txt 2012-04-28 18:08:28 UTC (rev 3095) @@ -11,7 +11,7 @@ lib/raidfile lib/crypto -lib/server qdbm +lib/server qdbm lib/crypto lib/compress lib/intercept From subversion at boxbackup.org Sat Apr 28 19:11:21 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:11:21 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3096 - in box/trunk/lib: crypto server Message-ID: <20120428181121.D1C531A3769@www.boxbackup.org> Author: chris Date: 2012-04-28 19:11:21 +0100 (Sat, 28 Apr 2012) New Revision: 3096 Added: box/trunk/lib/crypto/CryptoUtils.cpp box/trunk/lib/crypto/CryptoUtils.h Modified: box/trunk/lib/server/SSLLib.cpp box/trunk/lib/server/SSLLib.h box/trunk/lib/server/SocketStreamTLS.cpp box/trunk/lib/server/TLSContext.cpp Log: Move LogError out of server/SSLLib so we can use it in Crypto. Added: box/trunk/lib/crypto/CryptoUtils.cpp =================================================================== --- box/trunk/lib/crypto/CryptoUtils.cpp (rev 0) +++ box/trunk/lib/crypto/CryptoUtils.cpp 2012-04-28 18:11:21 UTC (rev 3096) @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CryptoUtils.cpp +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2012/04/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include +#include + +#include "CryptoUtils.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CryptoUtils::LogError(const char *) +// Purpose: Logs an error from the OpenSSL library +// Created: 2012/04/26 +// +// -------------------------------------------------------------------------- +std::string CryptoUtils::LogError(const std::string& rErrorDuringAction) +{ + unsigned long errcode; + char errname[256]; // SSL docs say at least 120 bytes + std::string firstError; + + while((errcode = ERR_get_error()) != 0) + { + ::ERR_error_string_n(errcode, errname, sizeof(errname)); + if(firstError.empty()) + { + firstError = errname; + } + BOX_ERROR("SSL or crypto error: " << rErrorDuringAction << + ": " << errname); + } + return firstError; +} + Added: box/trunk/lib/crypto/CryptoUtils.h =================================================================== --- box/trunk/lib/crypto/CryptoUtils.h (rev 0) +++ box/trunk/lib/crypto/CryptoUtils.h 2012-04-28 18:11:21 UTC (rev 3096) @@ -0,0 +1,27 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CryptoUtils.h +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2012/04/26 +// +// -------------------------------------------------------------------------- + +#ifndef CRYPTOUTILS__H +#define CRYPTOUTILS__H + +// -------------------------------------------------------------------------- +// +// Namespace +// Name: CryptoUtils +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +namespace CryptoUtils +{ + std::string LogError(const std::string& rErrorDuringAction); +}; + +#endif // CRYPTOUTILS__H + Modified: box/trunk/lib/server/SSLLib.cpp =================================================================== --- box/trunk/lib/server/SSLLib.cpp 2012-04-28 18:08:28 UTC (rev 3095) +++ box/trunk/lib/server/SSLLib.cpp 2012-04-28 18:11:21 UTC (rev 3096) @@ -18,6 +18,7 @@ #include #endif +#include "CryptoUtils.h" #include "SSLLib.h" #include "ServerException.h" @@ -39,8 +40,9 @@ { if(!::SSL_library_init()) { - LogError("initialising OpenSSL"); - THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError) + THROW_EXCEPTION_MESSAGE(ServerException, + SSLLibraryInitialisationError, + CryptoUtils::LogError("initialising OpenSSL")); } // More helpful error messages @@ -89,23 +91,3 @@ } -// -------------------------------------------------------------------------- -// -// Function -// Name: SSLLib::LogError(const char *) -// Purpose: Logs an error -// Created: 2003/08/06 -// -// -------------------------------------------------------------------------- -void SSLLib::LogError(const std::string& rErrorDuringAction) -{ - unsigned long errcode; - char errname[256]; // SSL docs say at least 120 bytes - while((errcode = ERR_get_error()) != 0) - { - ::ERR_error_string_n(errcode, errname, sizeof(errname)); - BOX_ERROR("SSL error while " << rErrorDuringAction << ": " << - errname); - } -} - Modified: box/trunk/lib/server/SSLLib.h =================================================================== --- box/trunk/lib/server/SSLLib.h 2012-04-28 18:08:28 UTC (rev 3095) +++ box/trunk/lib/server/SSLLib.h 2012-04-28 18:11:21 UTC (rev 3096) @@ -29,7 +29,6 @@ namespace SSLLib { void Initialise(); - void LogError(const std::string& rErrorDuringAction); }; #endif // SSLLIB__H Modified: box/trunk/lib/server/SocketStreamTLS.cpp =================================================================== --- box/trunk/lib/server/SocketStreamTLS.cpp 2012-04-28 18:08:28 UTC (rev 3095) +++ box/trunk/lib/server/SocketStreamTLS.cpp 2012-04-28 18:11:21 UTC (rev 3096) @@ -19,11 +19,12 @@ #include #endif +#include "BoxTime.h" +#include "CryptoUtils.h" +#include "ServerException.h" #include "SocketStreamTLS.h" #include "SSLLib.h" -#include "ServerException.h" #include "TLSContext.h" -#include "BoxTime.h" #include "MemLeakFindOn.h" @@ -124,7 +125,7 @@ mpBIO = ::BIO_new(::BIO_s_socket()); if(mpBIO == 0) { - SSLLib::LogError("creating socket bio"); + CryptoUtils::LogError("creating socket bio"); THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -135,7 +136,7 @@ mpSSL = ::SSL_new(rContext.GetRawContext()); if(mpSSL == 0) { - SSLLib::LogError("creating SSL object"); + CryptoUtils::LogError("creating SSL object"); THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -203,12 +204,12 @@ // Error occured if(IsServer) { - SSLLib::LogError("accepting connection"); + CryptoUtils::LogError("accepting connection"); THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } else { - SSLLib::LogError("connecting"); + CryptoUtils::LogError("connecting"); THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } } @@ -335,7 +336,7 @@ break; default: - SSLLib::LogError("reading"); + CryptoUtils::LogError("reading"); THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) break; } @@ -400,7 +401,7 @@ break; default: - SSLLib::LogError("writing"); + CryptoUtils::LogError("writing"); THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) break; } @@ -442,7 +443,7 @@ if(::SSL_shutdown(mpSSL) < 0) { - SSLLib::LogError("shutting down"); + CryptoUtils::LogError("shutting down"); THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) } Modified: box/trunk/lib/server/TLSContext.cpp =================================================================== --- box/trunk/lib/server/TLSContext.cpp 2012-04-28 18:08:28 UTC (rev 3095) +++ box/trunk/lib/server/TLSContext.cpp 2012-04-28 18:11:21 UTC (rev 3096) @@ -12,7 +12,7 @@ #define TLS_CLASS_IMPLEMENTATION_CPP #include -#include "TLSContext.h" +#include "CryptoUtils.h" #include "ServerException.h" #include "SSLLib.h" #include "TLSContext.h" @@ -77,14 +77,14 @@ { std::string msg = "loading certificates from "; msg += CertificatesFile; - SSLLib::LogError(msg); + CryptoUtils::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed) } if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1) { std::string msg = "loading private key from "; msg += PrivateKeyFile; - SSLLib::LogError(msg); + CryptoUtils::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed) } @@ -93,7 +93,7 @@ { std::string msg = "loading CA cert from "; msg += TrustedCAsFile; - SSLLib::LogError(msg); + CryptoUtils::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed) } @@ -105,7 +105,7 @@ // Setup allowed ciphers if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1) { - SSLLib::LogError("setting cipher list to " CIPHER_LIST); + CryptoUtils::LogError("setting cipher list to " CIPHER_LIST); THROW_EXCEPTION(ServerException, TLSSetCiphersFailed) } } From subversion at boxbackup.org Sat Apr 28 19:12:23 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:12:23 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3097 - box/trunk/lib/crypto Message-ID: <20120428181223.0F7E61A37C1@www.boxbackup.org> Author: chris Date: 2012-04-28 19:12:22 +0100 (Sat, 28 Apr 2012) New Revision: 3097 Modified: box/trunk/lib/crypto/CipherContext.cpp Log: Log errors from OpenSSL and clear the error queue to avoid bad state. Modified: box/trunk/lib/crypto/CipherContext.cpp =================================================================== --- box/trunk/lib/crypto/CipherContext.cpp 2012-04-28 18:11:21 UTC (rev 3096) +++ box/trunk/lib/crypto/CipherContext.cpp 2012-04-28 18:12:22 UTC (rev 3097) @@ -13,6 +13,7 @@ #include "CipherContext.h" #include "CipherDescription.h" #include "CipherException.h" +#include "CryptoUtils.h" #include "Random.h" #include "MemLeakFindOn.h" @@ -26,12 +27,12 @@ // // -------------------------------------------------------------------------- CipherContext::CipherContext() - : mInitialised(false), - mWithinTransform(false), - mPaddingOn(true) +: mInitialised(false), + mWithinTransform(false), + mPaddingOn(true), + mFunction(None) #ifdef HAVE_OLD_SSL - , mFunction(Decrypt), - mpDescription(0) +, mpDescription(0) #endif { } @@ -64,6 +65,28 @@ // -------------------------------------------------------------------------- // // Function +// Name: CipherContext::LogError(const std::string& operation) +// Purpose: Logs and clears any OpenSSL errors, returning the +// most recent error message for use in exception +// messages. +// +// It's essential to clear the OpenSSL error queue after +// ANY failed OpenSSL operation, because OpenSSL may +// decide that a later non-blocking read (returning -1 +// with errno == EAGAIN) is actually an error if there's +// any errors left in the queue. See SSL_get_error +// (called from SocketStreamTLS::Read) for the details. +// Created: 26/04/12 +// +// -------------------------------------------------------------------------- +std::string CipherContext::LogError(const std::string& operation) +{ + return CryptoUtils::LogError(operation); +} + +// -------------------------------------------------------------------------- +// +// Function // Name: CipherContext::Init(CipherContext::CipherFunction, const CipherDescription &) // Purpose: Initialises the context, specifying the direction for the encryption, and a // description of the cipher to use, it's keys, etc @@ -82,24 +105,29 @@ THROW_EXCEPTION(CipherException, BadArguments) } + // Store function for later + mFunction = Function; + // Initialise the cipher #ifndef HAVE_OLD_SSL EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does - if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, Function) != 1) + if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, + (mFunction == Encrypt) ? 1 : 0) != 1) #else - // Store function for later - mFunction = Function; - // Use old version of init call - if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, Function) != 1) + if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, + (mFunction == Encrypt) ? 1 : 0) != 1) #endif { - THROW_EXCEPTION(CipherException, EVPInitFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, + "Failed to initialise " << rDescription.GetFullName() + << "cipher: " << LogError("initialising cipher")); } try { + mCipherName = rDescription.GetFullName(); #ifndef HAVE_OLD_SSL // Let the description set up everything else rDescription.SetupParameters(&ctx); @@ -114,6 +142,9 @@ } catch(...) { + THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, + "Failed to configure " << mCipherName << " cipher: " << + LogError("configuring cipher")); EVP_CIPHER_CTX_cleanup(&ctx); throw; } @@ -174,7 +205,9 @@ // Initialise the cipher context again if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) { - THROW_EXCEPTION(CipherException, EVPInitFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, + "Failed to reset " << mCipherName << " cipher: " << + LogError("resetting cipher")); } // Mark as being within a transform @@ -227,7 +260,9 @@ int outLength = OutLength; if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) { - THROW_EXCEPTION(CipherException, EVPUpdateFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, + "Failed to " << GetFunction() << " (update) " << + mCipherName << " cipher: " << LogError(GetFunction())); } return outLength; @@ -273,9 +308,12 @@ // Do the transform int outLength = OutLength; #ifndef HAVE_OLD_SSL - if(EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) + if(EVP_CipherFinal(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) { - THROW_EXCEPTION(CipherException, EVPFinalFailure) + mWithinTransform = false; + THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, + "Failed to " << GetFunction() << " (final) " << + mCipherName << " cipher: " << LogError(GetFunction())); } #else OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); @@ -353,7 +391,8 @@ } } // Reinitialise the cipher for the next time around - if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, mFunction) != 1) + if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, + (mFunction == Encrypt) ? 1 : 0) != 1) { THROW_EXCEPTION(CipherException, EVPInitFailure) } @@ -451,37 +490,29 @@ // Do the entire block int outLength = 0; - try + + // Update + outLength = OutLength; + if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) { - // Update - outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) - { - THROW_EXCEPTION(CipherException, EVPUpdateFailure) - } - // Finalise - int outLength2 = OutLength - outLength; + THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, + "Failed to " << GetFunction() << " (update) " << + mCipherName << " cipher: " << LogError(GetFunction())); + } + + // Finalise + int outLength2 = OutLength - outLength; #ifndef HAVE_OLD_SSL - if(EVP_CipherFinal_ex(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) - { - THROW_EXCEPTION(CipherException, EVPFinalFailure) - } -#else - OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); -#endif - outLength += outLength2; + if(EVP_CipherFinal(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) + { + THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, + "Failed to " << GetFunction() << " (final) " << + mCipherName << " cipher: " << LogError(GetFunction())); } - catch(...) - { - // Finalise the context, so definately ready for the next caller - int outs = OutLength; -#ifndef HAVE_OLD_SSL - EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outs); #else - OldOpenSSLFinal((unsigned char*)pOutBuffer, outs); + OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); #endif - throw; - } + outLength += outLength2; return outLength; } @@ -531,7 +562,9 @@ // Set IV if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) { - THROW_EXCEPTION(CipherException, EVPInitFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, + "Failed to " << GetFunction() << " (set IV) " << + mCipherName << " cipher: " << LogError(GetFunction())); } #ifdef HAVE_OLD_SSL @@ -576,21 +609,8 @@ // Generate some random data Random::Generate(mGeneratedIV, ivLen); + SetIV(mGeneratedIV); - // Set IV - if(EVP_CipherInit(&ctx, NULL, NULL, mGeneratedIV, -1) != 1) - { - THROW_EXCEPTION(CipherException, EVPInitFailure) - } - -#ifdef HAVE_OLD_SSL - // Update description - if(mpDescription != 0) - { - mpDescription->SetIV(mGeneratedIV); - } -#endif - // Return the IV and it's length rLengthOut = ivLen; return mGeneratedIV; From subversion at boxbackup.org Sat Apr 28 19:13:19 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:13:19 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3098 - box/trunk/lib/crypto Message-ID: <20120428181319.856C01A3831@www.boxbackup.org> Author: chris Date: 2012-04-28 19:13:19 +0100 (Sat, 28 Apr 2012) New Revision: 3098 Modified: box/trunk/lib/crypto/CipherAES.h box/trunk/lib/crypto/CipherBlowfish.h box/trunk/lib/crypto/CipherContext.h box/trunk/lib/crypto/CipherDescription.h Log: Allow ciphers to identify themselves for debugging. Modified: box/trunk/lib/crypto/CipherAES.h =================================================================== --- box/trunk/lib/crypto/CipherAES.h 2012-04-28 18:12:22 UTC (rev 3097) +++ box/trunk/lib/crypto/CipherAES.h 2012-04-28 18:13:19 UTC (rev 3098) @@ -37,6 +37,15 @@ // Setup any other parameters virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; + virtual std::string GetCipherName() const + { + std::ostringstream out; + out << "AES"; + out << mKeyLength; + return out.str(); + } + virtual CipherMode GetCipherMode() const { return mMode; } + private: CipherDescription::CipherMode mMode; const void *mpKey; Modified: box/trunk/lib/crypto/CipherBlowfish.h =================================================================== --- box/trunk/lib/crypto/CipherBlowfish.h 2012-04-28 18:12:22 UTC (rev 3097) +++ box/trunk/lib/crypto/CipherBlowfish.h 2012-04-28 18:13:19 UTC (rev 3098) @@ -38,6 +38,15 @@ // Setup any other parameters virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; + virtual std::string GetCipherName() const + { + std::ostringstream out; + out << "AES"; + out << mKeyLength; + return out.str(); + } + virtual CipherMode GetCipherMode() const { return mMode; } + #ifdef HAVE_OLD_SSL CipherDescription *Clone() const; void SetIV(const void *pIV); Modified: box/trunk/lib/crypto/CipherContext.h =================================================================== --- box/trunk/lib/crypto/CipherContext.h 2012-04-28 18:12:22 UTC (rev 3097) +++ box/trunk/lib/crypto/CipherContext.h 2012-04-28 18:13:19 UTC (rev 3098) @@ -35,12 +35,15 @@ private: CipherContext(const CipherContext &); // no copying CipherContext &operator=(const CipherContext &); // no assignment +protected: + std::string LogError(const std::string& operation); public: typedef enum { - Decrypt = 0, - Encrypt = 1 + None = 0, + Decrypt, + Encrypt } CipherFunction; void Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription); @@ -61,6 +64,10 @@ const void *SetRandomIV(int &rLengthOut); void UsePadding(bool Padding = true); + const char* GetFunction() const + { + return (mFunction == Encrypt) ? "encrypt" : "decrypt"; + } #ifdef HAVE_OLD_SSL void OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut); @@ -72,8 +79,9 @@ bool mWithinTransform; bool mPaddingOn; uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; + CipherFunction mFunction; + std::string mCipherName; #ifdef HAVE_OLD_SSL - CipherFunction mFunction; CipherDescription *mpDescription; #endif }; Modified: box/trunk/lib/crypto/CipherDescription.h =================================================================== --- box/trunk/lib/crypto/CipherDescription.h 2012-04-28 18:12:22 UTC (rev 3097) +++ box/trunk/lib/crypto/CipherDescription.h 2012-04-28 18:13:19 UTC (rev 3098) @@ -34,7 +34,7 @@ // Return OpenSSL cipher object virtual const EVP_CIPHER *GetCipher() const = 0; - + // Setup any other parameters virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const = 0; @@ -47,6 +47,23 @@ Mode_OFB = 3 } CipherMode; + virtual std::string GetCipherName() const = 0; + virtual CipherMode GetCipherMode() const = 0; + virtual std::string GetFullName() const + { + std::ostringstream out; + out << GetCipherName() << "-"; + switch (GetCipherMode()) + { + case Mode_ECB: out << "ECB"; break; + case Mode_CBC: out << "CBC"; break; + case Mode_CFB: out << "CFB"; break; + case Mode_OFB: out << "OFB"; break; + default: out << "unknown"; + } + return out.str(); + } + #ifdef HAVE_OLD_SSL // For the old version of OpenSSL, we need to be able to store cipher descriptions. virtual CipherDescription *Clone() const = 0; From subversion at boxbackup.org Sat Apr 28 19:13:47 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:13:47 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3099 - box/trunk/lib/backupstore Message-ID: <20120428181347.B2F6B1A385B@www.boxbackup.org> Author: chris Date: 2012-04-28 19:13:47 +0100 (Sat, 28 Apr 2012) New Revision: 3099 Modified: box/trunk/lib/backupstore/BackupStoreDirectory.h Log: Cosmetic improvements in comments. Modified: box/trunk/lib/backupstore/BackupStoreDirectory.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreDirectory.h 2012-04-28 18:13:19 UTC (rev 3098) +++ box/trunk/lib/backupstore/BackupStoreDirectory.h 2012-04-28 18:13:47 UTC (rev 3099) @@ -268,12 +268,11 @@ box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); bool NameInUse(const BackupStoreFilename &rName); - // Don't use these functions in normal code! // For testing + // Don't use these functions in normal code! void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} - - // Debug and diagonistics + // Debug and diagnostics void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere private: From subversion at boxbackup.org Sat Apr 28 19:18:15 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:18:15 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3100 - box/trunk/lib/server Message-ID: <20120428181815.A63B61A388C@www.boxbackup.org> Author: chris Date: 2012-04-28 19:18:15 +0100 (Sat, 28 Apr 2012) New Revision: 3100 Modified: box/trunk/lib/server/makeprotocol.pl.in Log: Defend against exceptions during logging, e.g. CipherException if filename decrypt fails. Modified: box/trunk/lib/server/makeprotocol.pl.in =================================================================== --- box/trunk/lib/server/makeprotocol.pl.in 2012-04-28 18:13:47 UTC (rev 3099) +++ box/trunk/lib/server/makeprotocol.pl.in 2012-04-28 18:18:15 UTC (rev 3100) @@ -469,12 +469,28 @@ print CPP <<__E; void $cmd_class\::LogSysLog(const char *Action) const { - BOX_TRACE($log); + try + { + BOX_TRACE($log); + } + catch(std::exception &e) + { + BOX_WARNING("Failed to log command: " << Action << ": " << + e.what()); + } } void $cmd_class\::LogFile(const char *Action, FILE *File) const { std::ostringstream oss; - oss << $log; + try + { + oss << $log; + } + catch(std::exception &e) + { + oss << "Failed to log command: " << Action << ": " << + e.what(); + } ::fprintf(File, "%s\\n", oss.str().c_str()); ::fflush(File); } From subversion at boxbackup.org Sat Apr 28 19:18:59 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:18:59 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3101 - box/trunk/lib/common Message-ID: <20120428181859.454981A394F@www.boxbackup.org> Author: chris Date: 2012-04-28 19:18:59 +0100 (Sat, 28 Apr 2012) New Revision: 3101 Modified: box/trunk/lib/common/Box.h box/trunk/lib/common/Logging.cpp box/trunk/lib/common/Logging.h Log: Allow hiding specific exceptions to keep test output cleaner. Modified: box/trunk/lib/common/Box.h =================================================================== --- box/trunk/lib/common/Box.h 2012-04-28 18:18:15 UTC (rev 3100) +++ box/trunk/lib/common/Box.h 2012-04-28 18:18:59 UTC (rev 3101) @@ -105,7 +105,9 @@ #define THROW_EXCEPTION(type, subtype) \ { \ - if(!HideExceptionMessageGuard::ExceptionsHidden() \ + if((!HideExceptionMessageGuard::ExceptionsHidden() \ + && !HideSpecificExceptionGuard::IsHidden( \ + type::ExceptionType, type::subtype)) \ || Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ { \ std::auto_ptr guard; \ @@ -127,7 +129,9 @@ { \ std::ostringstream _box_throw_line; \ _box_throw_line << message; \ - if(!HideExceptionMessageGuard::ExceptionsHidden() \ + if((!HideExceptionMessageGuard::ExceptionsHidden() \ + && !HideSpecificExceptionGuard::IsHidden( \ + type::ExceptionType, type::subtype)) \ || Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ { \ std::auto_ptr guard; \ @@ -139,8 +143,9 @@ \ OPTIONAL_DO_BACKTRACE \ BOX_WARNING("Exception thrown: " \ - #type "(" #subtype ") (" << message << \ - ") at " __FILE__ "(" << __LINE__ << ")") \ + #type "(" #subtype ") (" << \ + _box_throw_line.str() << \ + ") at " __FILE__ ":" << __LINE__) \ } \ throw type(type::subtype, _box_throw_line.str()); \ } Modified: box/trunk/lib/common/Logging.cpp =================================================================== --- box/trunk/lib/common/Logging.cpp 2012-04-28 18:18:15 UTC (rev 3100) +++ box/trunk/lib/common/Logging.cpp 2012-04-28 18:18:59 UTC (rev 3101) @@ -46,6 +46,9 @@ Logging Logging::sGlobalLogging; //automatic initialisation std::string Logging::sProgramName; +HideSpecificExceptionGuard::SuppressedExceptions_t + HideSpecificExceptionGuard::sSuppressedExceptions; + int Logging::Guard::sGuardCount = 0; Log::Level Logging::Guard::sOriginalLevel = Log::INVALID; @@ -538,3 +541,18 @@ return output.str(); } + +bool HideSpecificExceptionGuard::IsHidden(int type, int subtype) +{ + for (SuppressedExceptions_t::iterator + i = sSuppressedExceptions.begin(); + i != sSuppressedExceptions.end(); i++) + { + if(i->first == type && i->second == subtype) + { + return true; + } + } + return false; +} + Modified: box/trunk/lib/common/Logging.h =================================================================== --- box/trunk/lib/common/Logging.h 2012-04-28 18:18:15 UTC (rev 3100) +++ box/trunk/lib/common/Logging.h 2012-04-28 18:18:59 UTC (rev 3101) @@ -10,6 +10,8 @@ #ifndef LOGGING__H #define LOGGING__H +#include + #include #include #include @@ -389,6 +391,30 @@ bool mOldHiddenState; }; +class HideSpecificExceptionGuard +{ + private: + std::pair mExceptionCode; + + public: + typedef std::vector > SuppressedExceptions_t; + static SuppressedExceptions_t sSuppressedExceptions; + + HideSpecificExceptionGuard(int type, int subtype) + : mExceptionCode(std::pair(type, subtype)) + { + sSuppressedExceptions.push_back(mExceptionCode); + } + ~HideSpecificExceptionGuard() + { + SuppressedExceptions_t::reverse_iterator i = + sSuppressedExceptions.rbegin(); + assert(*i == mExceptionCode); + sSuppressedExceptions.pop_back(); + } + static bool IsHidden(int type, int subtype); +}; + std::string PrintEscapedBinaryData(const std::string& rInput); #endif // LOGGING__H From subversion at boxbackup.org Sat Apr 28 19:19:53 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:19:53 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3102 - box/trunk/bin/bbackupd Message-ID: <20120428181953.D0EEE1A3970@www.boxbackup.org> Author: chris Date: 2012-04-28 19:19:53 +0100 (Sat, 28 Apr 2012) New Revision: 3102 Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp box/trunk/bin/bbackupd/BackupDaemon.h Log: Allow BackupDaemon user to reset state for testing. Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.cpp 2012-04-28 18:18:59 UTC (rev 3101) +++ box/trunk/bin/bbackupd/BackupDaemon.cpp 2012-04-28 18:19:53 UTC (rev 3102) @@ -721,12 +721,7 @@ DeleteCorruptBerkelyDbFiles(); } - // Clear state data - // Go back to beginning of time - mLastSyncTime = 0; - mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything - DeleteAllLocations(); - DeleteAllIDMaps(); + ResetCachedState(); // Handle restart? if(StopRun()) @@ -789,6 +784,16 @@ OnBackupFinish(); } +void BackupDaemon::ResetCachedState() +{ + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); +} + void BackupDaemon::RunSyncNow() { // Delete the serialised store object file, Modified: box/trunk/bin/bbackupd/BackupDaemon.h =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.h 2012-04-28 18:18:59 UTC (rev 3101) +++ box/trunk/bin/bbackupd/BackupDaemon.h 2012-04-28 18:19:53 UTC (rev 3102) @@ -117,6 +117,7 @@ void InitCrypto(); void RunSyncNowWithExceptionHandling(); void RunSyncNow(); + void ResetCachedState(); void OnBackupStart(); void OnBackupFinish(); // TouchFileInWorkingDir is only here for use by Boxi. From subversion at boxbackup.org Sat Apr 28 19:20:39 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:20:39 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3103 - box/trunk/bin/bbackupquery Message-ID: <20120428182039.22EED1A39E9@www.boxbackup.org> Author: chris Date: 2012-04-28 19:20:38 +0100 (Sat, 28 Apr 2012) New Revision: 3103 Modified: box/trunk/bin/bbackupquery/BackupQueries.cpp box/trunk/bin/bbackupquery/BackupQueries.h Log: Change BackupQueries List() to use C++ streams for output. Modified: box/trunk/bin/bbackupquery/BackupQueries.cpp =================================================================== --- box/trunk/bin/bbackupquery/BackupQueries.cpp 2012-04-28 18:19:53 UTC (rev 3102) +++ box/trunk/bin/bbackupquery/BackupQueries.cpp 2012-04-28 18:20:38 UTC (rev 3103) @@ -50,6 +50,7 @@ #include "SelfFlushingStream.h" #include "Utils.h" #include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" #include "MemLeakFindOn.h" @@ -355,7 +356,8 @@ // Created: 2003/10/10 // // -------------------------------------------------------------------------- -void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel) +void BackupQueries::List(int64_t DirID, const std::string &rListRoot, + const bool *opts, bool FirstLevel, std::ostream &out) { // Generate exclude flags int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; @@ -366,11 +368,11 @@ try { mrConnection.QueryListDirectory( - DirID, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - // both files and directories - excludeFlags, - true /* want attributes */); + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // both files and directories + excludeFlags, + true /* want attributes */); } catch (std::exception &e) { @@ -385,7 +387,6 @@ return; } - // Retrieve the directory from the stream following BackupStoreDirectory dir; std::auto_ptr dirstream(mrConnection.ReceiveStream()); @@ -403,11 +404,9 @@ if(!opts[LIST_OPTION_NOOBJECTID]) { // add object ID to line -#ifdef _MSC_VER - printf("%08I64x ", (int64_t)en->GetObjectID()); -#else - printf("%08llx ", (long long)en->GetObjectID()); -#endif + out << std::hex << std::internal << std::setw(8) << + std::setfill('0') << en->GetObjectID() << + std::dec << " "; } // Flags? @@ -434,44 +433,40 @@ // terminate *(f++) = ' '; *(f++) = '\0'; - printf("%s", displayflags); + out << displayflags; if(en_flags != 0) { - printf("[ERROR: Entry has additional flags set] "); + out << "[ERROR: Entry has additional flags set] "; } } if(opts[LIST_OPTION_TIMES_UTC]) { // Show UTC times... - printf("%s ", GetTimeString(*en, false, - opts[LIST_OPTION_TIMES_ATTRIBS]).c_str()); + out << GetTimeString(*en, false, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; } if(opts[LIST_OPTION_TIMES_LOCAL]) { // Show local times... - printf("%s ", GetTimeString(*en, true, - opts[LIST_OPTION_TIMES_ATTRIBS]).c_str()); + out << GetTimeString(*en, true, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; } if(opts[LIST_OPTION_DISPLAY_HASH]) { -#ifdef _MSC_VER - printf("%016I64x ", (int64_t)en->GetAttributesHash()); -#else - printf("%016llx ", (long long)en->GetAttributesHash()); -#endif + out << std::hex << std::internal << std::setw(16) << + std::setfill('0') << en->GetAttributesHash() << + std::dec; } if(opts[LIST_OPTION_SIZEINBLOCKS]) { -#ifdef _MSC_VER - printf("%05I64d ", (int64_t)en->GetSizeInBlocks()); -#else - printf("%05lld ", (long long)en->GetSizeInBlocks()); -#endif + out << std::internal << std::setw(5) << + std::setfill('0') << en->GetSizeInBlocks() << + " "; } // add name @@ -481,30 +476,38 @@ std::string listRootDecoded; if(!ConvertUtf8ToConsole(rListRoot.c_str(), listRootDecoded)) return; - printf("%s/", listRootDecoded.c_str()); + out << listRootDecoded << "/"; #else - printf("%s/", rListRoot.c_str()); + out << rListRoot << "/"; #endif } + std::string fileName; + try + { + fileName = clear.GetClearFilename(); + } + catch(CipherException &e) + { + fileName = ""; + } + #ifdef WIN32 + std::string fileNameUtf8 = fileName; + if(!ConvertUtf8ToConsole(fileNameUtf8, fileName)) { - std::string fileName; - if(!ConvertUtf8ToConsole( - clear.GetClearFilename().c_str(), fileName)) - return; - printf("%s", fileName.c_str()); + fileName = fileNameUtf8 + " [convert encoding failed]"; } -#else - printf("%s", clear.GetClearFilename().c_str()); #endif + + out << fileName; if(!en->GetName().IsEncrypted()) { - printf("[FILENAME NOT ENCRYPTED]"); + out << " [FILENAME NOT ENCRYPTED]"; } - printf("\n"); + out << std::endl; // Directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) @@ -515,7 +518,9 @@ std::string subroot(rListRoot); if(!FirstLevel) subroot += '/'; subroot += clear.GetClearFilename(); - List(en->GetObjectID(), subroot, opts, false /* not the first level to list */); + List(en->GetObjectID(), subroot, opts, + false /* not the first level to list */, + out); } } } Modified: box/trunk/bin/bbackupquery/BackupQueries.h =================================================================== --- box/trunk/bin/bbackupquery/BackupQueries.h 2012-04-28 18:19:53 UTC (rev 3102) +++ box/trunk/bin/bbackupquery/BackupQueries.h 2012-04-28 18:20:38 UTC (rev 3103) @@ -10,8 +10,9 @@ #ifndef BACKUPQUERIES__H #define BACKUPQUERIES__H +#include +#include #include -#include #include "BoxTime.h" #include "BoxBackupCompareParams.h" @@ -85,9 +86,12 @@ // Return code? int GetReturnCode() {return mReturnCode;} + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, + bool FirstLevel, std::ostream &out = std::cout); + void CommandList(const std::vector &args, const bool *opts); + private: // Commands - void CommandList(const std::vector &args, const bool *opts); void CommandChangeDir(const std::vector &args, const bool *opts); void CommandChangeLocalDir(const std::vector &args); void CommandGetObject(const std::vector &args, const bool *opts); @@ -102,10 +106,6 @@ int64_t HardLimit, int32_t BlockSize, bool MachineReadable); void CommandHelp(const std::vector &args); - // Implementations - void List(int64_t DirID, const std::string &rListRoot, const bool *opts, - bool FirstLevel); - public: class CompareParams : public BoxBackupCompareParams { From subversion at boxbackup.org Sat Apr 28 19:21:22 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:21:22 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3104 - box/trunk/bin/bbstored Message-ID: <20120428182122.1AD7A1A3A13@www.boxbackup.org> Author: chris Date: 2012-04-28 19:21:21 +0100 (Sat, 28 Apr 2012) New Revision: 3104 Modified: box/trunk/bin/bbstored/BackupStoreDaemon.h Log: Make BackupStoreDaemon::RunHousekeepingIfNeeded public to allow tests to call it. Modified: box/trunk/bin/bbstored/BackupStoreDaemon.h =================================================================== --- box/trunk/bin/bbstored/BackupStoreDaemon.h 2012-04-28 18:20:38 UTC (rev 3103) +++ box/trunk/bin/bbstored/BackupStoreDaemon.h 2012-04-28 18:21:21 UTC (rev 3104) @@ -69,6 +69,7 @@ public: // HousekeepingInterface implementation virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); + void RunHousekeepingIfNeeded(); private: BackupStoreAccountDatabase *mpAccountDatabase; @@ -83,7 +84,6 @@ virtual void OnIdle(); void HousekeepingInit(); - void RunHousekeepingIfNeeded(); int64_t mLastHousekeepingRun; public: From subversion at boxbackup.org Sat Apr 28 19:21:55 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:21:55 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3105 - box/trunk/lib/common Message-ID: <20120428182155.DECC21A3A41@www.boxbackup.org> Author: chris Date: 2012-04-28 19:21:55 +0100 (Sat, 28 Apr 2012) New Revision: 3105 Modified: box/trunk/lib/common/ZeroStream.cpp Log: Fix bug that caused sending a ZeroStream to end early, breaking protocol. Modified: box/trunk/lib/common/ZeroStream.cpp =================================================================== --- box/trunk/lib/common/ZeroStream.cpp 2012-04-28 18:21:21 UTC (rev 3104) +++ box/trunk/lib/common/ZeroStream.cpp 2012-04-28 18:21:55 UTC (rev 3105) @@ -152,7 +152,7 @@ // -------------------------------------------------------------------------- bool ZeroStream::StreamDataLeft() { - return false; + return (BytesLeftToRead() > 0); } // -------------------------------------------------------------------------- From subversion at boxbackup.org Sat Apr 28 19:25:51 2012 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Apr 2012 19:25:51 +0100 (BST) Subject: [Box Backup-commit] COMMIT r3106 - box/trunk/lib/backupstore Message-ID: <20120428182551.182FC1A3AAB@www.boxbackup.org> Author: chris Date: 2012-04-28 19:25:50 +0100 (Sat, 28 Apr 2012) New Revision: 3106 Modified: box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp box/trunk/lib/backupstore/BackupStoreFilenameClear.h Log: Move private variables out of hidden namespace to help debugging. Modified: box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp 2012-04-28 18:21:55 UTC (rev 3105) +++ box/trunk/lib/backupstore/BackupStoreFilenameClear.cpp 2012-04-28 18:25:50 UTC (rev 3106) @@ -17,13 +17,9 @@ #include "MemLeakFindOn.h" -// Hide private variables from the rest of the world -namespace -{ - int sEncodeMethod = BackupStoreFilename::Encoding_Clear; - CipherContext sBlowfishEncrypt; - CipherContext sBlowfishDecrypt; -} +int BackupStoreFilenameClear::sEncodeMethod = BackupStoreFilename::Encoding_Clear; +CipherContext BackupStoreFilenameClear::sBlowfishEncrypt; +CipherContext BackupStoreFilenameClear::sBlowfishDecrypt; // -------------------------------------------------------------------------- // Modified: box/trunk/lib/backupstore/BackupStoreFilenameClear.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreFilenameClear.h 2012-04-28 18:21:55 UTC (rev 3105) +++ box/trunk/lib/backupstore/BackupStoreFilenameClear.h 2012-04-28 18:25:50 UTC (rev 3106) @@ -11,9 +11,8 @@ #define BACKUPSTOREFILENAMECLEAR__H #include "BackupStoreFilename.h" +#include "CipherContext.h" -class CipherContext; - // -------------------------------------------------------------------------- // // Class @@ -54,6 +53,9 @@ private: mutable BackupStoreFilename_base mClearFilename; + static CipherContext sBlowfishEncrypt; + static CipherContext sBlowfishDecrypt; + static int sEncodeMethod; }; #endif // BACKUPSTOREFILENAMECLEAR__H