From subversion at boxbackup.org Wed Jun 10 20:46:02 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:02 GMT Subject: [Box Backup-commit] COMMIT r3574 - box/trunk/lib/backupstore Message-ID: <201506101946.t5AJk2HQ004924@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:01 +0000 (Wed, 10 Jun 2015) New Revision: 3574 Modified: box/trunk/lib/backupstore/BackupStoreFile.cpp Log: Initialise ModificationTime when uploading complete file. Modified: box/trunk/lib/backupstore/BackupStoreFile.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreFile.cpp 2015-06-10 19:45:58 UTC (rev 3573) +++ box/trunk/lib/backupstore/BackupStoreFile.cpp 2015-06-10 19:46:01 UTC (rev 3574) @@ -1589,7 +1589,7 @@ else { pStream = BackupStoreFile::EncodeFile(LocalFilename, - DirectoryObjectID, StoreFilename); + DirectoryObjectID, StoreFilename, &ModificationTime); } std::auto_ptr upload(pStream.release()); From subversion at boxbackup.org Wed Jun 10 20:45:59 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:45:59 GMT Subject: [Box Backup-commit] COMMIT r3573 - in box/trunk: . infrastructure lib/server Message-ID: <201506101945.t5AJjwfx004903@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:45:58 +0000 (Wed, 10 Jun 2015) New Revision: 3573 Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp box/trunk/lib/server/Daemon.cpp box/trunk/modules.txt Log: Add command-line help to test executables. Modified: box/trunk/infrastructure/buildenv-testmain-template.cpp =================================================================== --- box/trunk/infrastructure/buildenv-testmain-template.cpp 2015-05-16 13:07:49 UTC (rev 3572) +++ box/trunk/infrastructure/buildenv-testmain-template.cpp 2015-06-10 19:45:58 UTC (rev 3573) @@ -41,10 +41,12 @@ #endif #include +#include #include #include #include "box_getopt.h" +#include "depot.h" #include "Logging.h" #include "Test.h" #include "Timer.h" @@ -237,6 +239,23 @@ #endif +int Usage(const std::string& ProgramName) +{ + std::cout << + "(built with QDBM " << dpversion << ")\n" + "\n" + "Usage: " << ProgramName << " [options]\n" + "\n" + "Options:\n" + " -c/--bbackupd-args Arguments to pass to bbackupd/BackupDaemon\n" + " -s/--bbstored-args Arguments to pass to bbstored/BackupStoreDaemon\n" + " -d/--test-daemon-args Arguments to pass to TestDaemon\n" + " -e/--execute-only Execute only specific named test, can repeat\n" + " -h/--help Show this command-line help\n" + << Logging::OptionParser::GetUsageString(); + return 0; +} + int main(int argc, char * const * argv) { // Start memory leak testing @@ -250,11 +269,12 @@ { "bbstored-args", required_argument, NULL, 's' }, { "test-daemon-args", required_argument, NULL, 'd' }, { "execute-only", required_argument, NULL, 'e' }, + { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; int c; - std::string options("c:d:e:s:"); + std::string options("c:d:e:hs:"); options += Logging::OptionParser::GetOptionString(); Logging::OptionParser LogLevel; @@ -289,6 +309,12 @@ } break; + case 'h': + { + return Usage(argv[0]); + } + break; + case 's': { bbstored_args += " "; Modified: box/trunk/lib/server/Daemon.cpp =================================================================== --- box/trunk/lib/server/Daemon.cpp 2015-05-16 13:07:49 UTC (rev 3572) +++ box/trunk/lib/server/Daemon.cpp 2015-06-10 19:45:58 UTC (rev 3573) @@ -134,8 +134,8 @@ " -k Keep console open after fork, keep writing log messages to it\n" " -K Stop writing log messages to console while daemon is running\n" " -o Log to a file, defaults to maximum verbosity\n" - " -O Set file log verbosity to error/warning/notice/info/trace/everything\n" << - Logging::OptionParser::GetUsageString(); + " -O Set file log verbosity to error/warning/notice/info/trace/everything\n" + << Logging::OptionParser::GetUsageString(); } // -------------------------------------------------------------------------- Modified: box/trunk/modules.txt =================================================================== --- box/trunk/modules.txt 2015-05-16 13:07:49 UTC (rev 3572) +++ box/trunk/modules.txt 2015-06-10 19:45:58 UTC (rev 3573) @@ -15,11 +15,11 @@ lib/compress lib/intercept -test/common -test/crypto lib/crypto -test/compress lib/compress -test/raidfile lib/raidfile lib/intercept -test/basicserver lib/server +test/common qdbm +test/crypto qdbm lib/crypto +test/compress qdbm lib/compress +test/raidfile qdbm lib/raidfile lib/intercept +test/basicserver qdbm lib/server # IF_DISTRIBUTION(boxbackup) From subversion at boxbackup.org Wed Jun 10 20:46:04 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:04 GMT Subject: [Box Backup-commit] COMMIT r3575 - box/trunk/lib/common Message-ID: <201506101946.t5AJk4eb004942@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:04 +0000 (Wed, 10 Jun 2015) New Revision: 3575 Modified: box/trunk/lib/common/Logging.cpp Log: Add usage example for -L option, showing how module names were compiled in Modified: box/trunk/lib/common/Logging.cpp =================================================================== --- box/trunk/lib/common/Logging.cpp 2015-06-10 19:46:01 UTC (rev 3574) +++ box/trunk/lib/common/Logging.cpp 2015-06-10 19:46:04 UTC (rev 3575) @@ -715,6 +715,7 @@ { return " -L Filter out log messages except from specified file, can repeat\n" + " (for example, -L " __FILE__ ")\n" " -N Truncate log file at startup and on backup start\n" " -P Show process ID (PID) in console output\n" " -q Run more quietly, reduce verbosity level by one, can repeat\n" From subversion at boxbackup.org Wed Jun 10 20:46:11 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:11 GMT Subject: [Box Backup-commit] COMMIT r3577 - box/trunk/lib/common Message-ID: <201506101946.t5AJkBu4004978@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:11 +0000 (Wed, 10 Jun 2015) New Revision: 3577 Modified: box/trunk/lib/common/Archive.h Log: Add missing timeouts to Archive Read() calls. Modified: box/trunk/lib/common/Archive.h =================================================================== --- box/trunk/lib/common/Archive.h 2015-06-10 19:46:08 UTC (rev 3576) +++ box/trunk/lib/common/Archive.h 2015-06-10 19:46:11 UTC (rev 3577) @@ -161,7 +161,7 @@ void Read(std::string &rItemOut) { int size; - Read(size); + Read(size, mTimeout); // Assume most strings are relatively small char buf[256]; From subversion at boxbackup.org Wed Jun 10 20:46:15 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:15 GMT Subject: [Box Backup-commit] COMMIT r3578 - box/trunk/lib/common Message-ID: <201506101946.t5AJkFm4004998@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:15 +0000 (Wed, 10 Jun 2015) New Revision: 3578 Modified: box/trunk/lib/common/Archive.h Log: Add support for optional items in Archives. If the item is not present (the end of the Archive is reached instead) then a default value is returned instead of throwing an exception. Modified: box/trunk/lib/common/Archive.h =================================================================== --- box/trunk/lib/common/Archive.h 2015-06-10 19:46:11 UTC (rev 3577) +++ box/trunk/lib/common/Archive.h 2015-06-10 19:46:15 UTC (rev 3578) @@ -158,10 +158,35 @@ Read(privItem); rItemOut = privItem; } + void ReadIfPresent(std::string &rItemOut, const std::string& ValueIfNotPresent) + { + Read(rItemOut, &ValueIfNotPresent); + } void Read(std::string &rItemOut) { + Read(rItemOut, NULL); + } +private: + void Read(std::string &rItemOut, const std::string* pValueIfNotPresent) + { int size; - Read(size, mTimeout); + int bytesRead; + if(!mrStream.ReadFullBuffer(&size, sizeof(size), &bytesRead, mTimeout)) + { + if(bytesRead == 0 && pValueIfNotPresent != NULL) + { + // item is simply not present + rItemOut = *pValueIfNotPresent; + return; + } + else + { + // bad number of remaining bytes + THROW_EXCEPTION(CommonException, + ArchiveBlockIncompleteRead) + } + } + size = ntohl(size); // Assume most strings are relatively small char buf[256]; From subversion at boxbackup.org Wed Jun 10 20:46:17 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:17 GMT Subject: [Box Backup-commit] COMMIT r3579 - box/trunk/lib/common Message-ID: <201506101946.t5AJkH58005016@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:17 +0000 (Wed, 10 Jun 2015) New Revision: 3579 Modified: box/trunk/lib/common/Guards.h Log: Allow MemoryBlockGuard to guard a block passed in (already malloced). Modified: box/trunk/lib/common/Guards.h =================================================================== --- box/trunk/lib/common/Guards.h 2015-06-10 19:46:15 UTC (rev 3578) +++ box/trunk/lib/common/Guards.h 2015-06-10 19:46:17 UTC (rev 3579) @@ -77,7 +77,7 @@ { public: MemoryBlockGuard(int BlockSize) - : mpBlock(::malloc(BlockSize)) + : mpBlock(::malloc(BlockSize)) { if(mpBlock == 0) { @@ -84,6 +84,15 @@ throw std::bad_alloc(); } } + + MemoryBlockGuard(void *pBlock) + : mpBlock(pBlock) + { + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + } ~MemoryBlockGuard() { From subversion at boxbackup.org Wed Jun 10 20:46:09 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:09 GMT Subject: [Box Backup-commit] COMMIT r3576 - box/trunk/lib/common Message-ID: <201506101946.t5AJk9mH004960@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:08 +0000 (Wed, 10 Jun 2015) New Revision: 3576 Modified: box/trunk/lib/common/NamedLock.cpp Log: Fix logging of NamedLock acquisition for O_EXLOCK method. Modified: box/trunk/lib/common/NamedLock.cpp =================================================================== --- box/trunk/lib/common/NamedLock.cpp 2015-06-10 19:46:04 UTC (rev 3575) +++ box/trunk/lib/common/NamedLock.cpp 2015-06-10 19:46:08 UTC (rev 3576) @@ -77,99 +77,98 @@ mFileName = rFilename; // See if the lock can be got + int flags = O_WRONLY | O_CREAT | O_TRUNC; + #if HAVE_DECL_O_EXLOCK + flags |= O_NONBLOCK | O_EXLOCK; BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXLOCK"); - int fd = ::open(rFilename.c_str(), - O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); - if(fd != -1) - { - // Got a lock, lovely - mFileDescriptor = fd; - return true; - } - - // Failed. Why? - if(errno != EWOULDBLOCK) - { - // Not the expected error - THROW_SYS_FILE_ERROR("Failed to open lockfile", rFilename, - CommonException, OSFileError); - } - - return false; -#else - int flags = O_WRONLY | O_CREAT | O_TRUNC; - -# if !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK +#elif !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK // We have no other way to get a lock, so all we can do is fail if // the file already exists, and take the risk of stale locks. flags |= O_EXCL; BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXCL"); -# else +#else BOX_TRACE("Trying to create lockfile " << rFilename << " without special flags"); -# endif +#endif int fd = ::open(rFilename.c_str(), flags, mode); if(fd == -1) - { - if(errno == EEXIST && (flags & O_EXCL)) +#if HAVE_DECL_O_EXLOCK + { // if() + if(errno == EWOULDBLOCK) { // Lockfile already exists, and we tried to open it // exclusively, which means we failed to lock it. - BOX_NOTICE("Failed to lock lockfile with O_EXCL: " << rFilename + BOX_NOTICE("Failed to lock lockfile with O_EXLOCK: " << rFilename << ": already locked by another process?"); return false; } else { - THROW_SYS_FILE_ERROR("Failed to open lockfile", + THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXLOCK", rFilename, CommonException, OSFileError); } } - - try - { -#ifdef HAVE_FLOCK - BOX_TRACE("Trying to lock lockfile " << rFilename << " using flock()"); - if(::flock(fd, LOCK_EX | LOCK_NB) != 0) - { - if(errno == EWOULDBLOCK) +#else // !HAVE_DECL_O_EXLOCK + { // if() + if(errno == EEXIST && (flags & O_EXCL)) { - ::close(fd); - BOX_NOTICE("Failed to lock lockfile with flock(): " << rFilename - << ": already locked by another process"); + // Lockfile already exists, and we tried to open it + // exclusively, which means we failed to lock it. + BOX_NOTICE("Failed to lock lockfile with O_EXCL: " << rFilename + << ": already locked by another process?"); return false; } else { - THROW_SYS_FILE_ERROR("Failed to lock lockfile with flock()", + THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXCL", rFilename, CommonException, OSFileError); } } -#elif HAVE_DECL_F_SETLK - struct flock desc; - desc.l_type = F_WRLCK; - desc.l_whence = SEEK_SET; - desc.l_start = 0; - desc.l_len = 0; - BOX_TRACE("Trying to lock lockfile " << rFilename << " using fcntl()"); - if(::fcntl(fd, F_SETLK, &desc) != 0) + + try { - if(errno == EAGAIN) +# ifdef HAVE_FLOCK + BOX_TRACE("Trying to lock lockfile " << rFilename << " using flock()"); + if(::flock(fd, LOCK_EX | LOCK_NB) != 0) { - ::close(fd); - BOX_NOTICE("Failed to lock lockfile with fcntl(): " << rFilename - << ": already locked by another process"); - return false; + if(errno == EWOULDBLOCK) + { + ::close(fd); + BOX_NOTICE("Failed to lock lockfile with flock(): " << rFilename + << ": already locked by another process"); + return false; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile with flock()", + rFilename, CommonException, OSFileError); + } } - else +# elif HAVE_DECL_F_SETLK + struct flock desc; + desc.l_type = F_WRLCK; + desc.l_whence = SEEK_SET; + desc.l_start = 0; + desc.l_len = 0; + BOX_TRACE("Trying to lock lockfile " << rFilename << " using fcntl()"); + if(::fcntl(fd, F_SETLK, &desc) != 0) { - THROW_SYS_FILE_ERROR("Failed to lock lockfile with fcntl()", - rFilename, CommonException, OSFileError); + if(errno == EAGAIN) + { + ::close(fd); + BOX_NOTICE("Failed to lock lockfile with fcntl(): " << rFilename + << ": already locked by another process"); + return false; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile with fcntl()", + rFilename, CommonException, OSFileError); + } } +# endif } -#endif - } catch(BoxException &e) { ::close(fd); @@ -176,6 +175,7 @@ BOX_NOTICE("Failed to lock lockfile " << rFilename << ": " << e.what()); throw; } +#endif // HAVE_DECL_O_EXLOCK if(!FileExists(rFilename)) { @@ -190,7 +190,6 @@ BOX_TRACE("Successfully locked lockfile " << rFilename); return true; -#endif } // -------------------------------------------------------------------------- From subversion at boxbackup.org Wed Jun 10 20:46:20 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:20 GMT Subject: [Box Backup-commit] COMMIT r3580 - box/trunk/bin/bbackupd Message-ID: <201506101946.t5AJkKmH005036@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:20 +0000 (Wed, 10 Jun 2015) New Revision: 3580 Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h Log: Switch inode database to storing values using Archive. This allows us to store the local path to the tracked file in the database, and will allow us to extend the storage format again in future if needed. Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2015-06-10 19:46:17 UTC (rev 3579) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2015-06-10 19:46:20 UTC (rev 3580) @@ -207,8 +207,8 @@ { BackupClientInodeToIDMap &idMap( rParams.mrContext.GetNewIDMap()); - idMap.AddToMap(dest_st.st_ino, mObjectID, - ContainingDirectoryID); + idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); } // Add attributes to checksum currentStateChecksum.Add(&dest_st.st_mode, @@ -1227,18 +1227,8 @@ BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); // Need to get an ID from somewhere... - if(latestObjectID != 0) + if(latestObjectID == 0) { - // Use this one - BOX_TRACE("Storing uploaded file ID " << - inodeNum << " (" << nonVssFilePath << ") " - "in ID map as object " << - latestObjectID << " with parent " << - mObjectID); - idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */); - } - else - { // Don't know it -- haven't sent anything to the store, and didn't get a listing. // Look it up in the current map, and if it's there, use that. const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); @@ -1263,13 +1253,23 @@ // into it. However, in a long running process this may happen occasionally and // not indicate anything wrong. // Run the release version for real life use, where this check is not made. - BOX_TRACE("Storing found file ID " << inodeNum << - " (" << nonVssFilePath << ") in ID map as " - "object " << objid << " with parent " << mObjectID); - idMap.AddToMap(inodeNum, objid, - mObjectID /* containing directory */); + + latestObjectID = objid; } } + + if(latestObjectID != 0) + { + BOX_TRACE("Storing uploaded file ID " << + inodeNum << " (" << nonVssFilePath << ") " + "in ID map as object " << + latestObjectID << " with parent " << + mObjectID); + idMap.AddToMap(inodeNum, latestObjectID, + mObjectID /* containing directory */, + nonVssFilePath); + } + } if (fileSynced) Modified: box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp 2015-06-10 19:46:17 UTC (rev 3579) +++ box/trunk/bin/bbackupd/BackupClientInodeToIDMap.cpp 2015-06-10 19:46:20 UTC (rev 3580) @@ -16,15 +16,16 @@ #include "BackupClientInodeToIDMap.h" #undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION +#include "Archive.h" #include "BackupStoreException.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "autogen_CommonException.h" #include "MemLeakFindOn.h" -typedef struct -{ - int64_t mObjectID; - int64_t mInDirectory; -} IDBRecord; +#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" +#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 #define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) @@ -118,12 +119,58 @@ if(!mpDepot) { - BOX_WARNING(BOX_DBM_MESSAGE("Failed to open inode " - "database: " << mFilename)); THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, BOX_DBM_MESSAGE("Failed to open inode database: " << mFilename)); } + + const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; + int32_t version = 0; + + if(CreateNew) + { + version = BOX_DBM_INODE_DB_VERSION_CURRENT; + + int ret = dpput(mpDepot, version_key, strlen(version_key), + (char *)(&version), sizeof(version), DP_DKEEP); + + if(!ret) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to write version number to inode " + "database: " << mFilename)); + } + } + else + { + int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, + sizeof(version), (char *)(&version)); + + if(ret == -1) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Missing version number in inode database. Perhaps it " + "needs to be recreated: " << mFilename); + } + + if(ret != sizeof(version)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong size version number in inode database: expected " + << sizeof(version) << " bytes but found " << ret); + } + + if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong version number in inode database: expected " << + BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << + version << ". Perhaps it needs to be recreated: " << + mFilename); + } + + // By this point the version number has been checked and is OK. + } // Read only flag mReadOnly = ReadOnly; @@ -175,7 +222,7 @@ // // -------------------------------------------------------------------------- void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, - int64_t InDirectory) + int64_t InDirectory, const std::string& LocalPath) { if(mReadOnly) { @@ -190,12 +237,15 @@ ASSERT_DBM_OPEN(); // Setup structures - IDBRecord rec; - rec.mObjectID = ObjectID; - rec.mInDirectory = InDirectory; + CollectInBufferStream buf; + Archive arc(buf, IOStream::TimeOutInfinite); + arc.WriteExact((uint64_t)ObjectID); + arc.WriteExact((uint64_t)InDirectory); + arc.Write(LocalPath); + buf.SetForReading(); ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - (const char *)&rec, sizeof(rec), DP_DOVER), + (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), "Failed to add record to inode database", mFilename, BackupStoreException, BerkelyDBFailure); } @@ -211,8 +261,8 @@ // Created: 11/11/03 // // -------------------------------------------------------------------------- -bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, - int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const +bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut) const { if(mEmpty) { @@ -226,20 +276,45 @@ } ASSERT_DBM_OPEN(); - - IDBRecord rec; - - if(dpgetwb(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - 0, sizeof(IDBRecord), (char *)&rec) == -1) + int size; + char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + 0, -1, &size); + if(data == NULL) { // key not in file return false; } - + + // Free data automatically when the guard goes out of scope. + MemoryBlockGuard guard(data); + MemBlockStream stream(data, size); + Archive arc(stream, IOStream::TimeOutInfinite); + // Return data - rObjectIDOut = rec.mObjectID; - rInDirectoryOut = rec.mInDirectory; + try + { + arc.Read(rObjectIDOut); + arc.Read(rInDirectoryOut); + if(pLocalPathOut) + { + arc.Read(*pLocalPathOut); + } + } + catch(CommonException &e) + { + if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) + { + THROW_FILE_ERROR("Failed to lookup record in inode database: " + << InodeRef << ": not enough data in record", mFilename, + BackupStoreException, BerkelyDBFailure); + // Need to throw precisely that exception to ensure that the + // invalid database is deleted, so that we don't hit the same + // error next time. + } + throw; + } + // Found return true; } Modified: box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h =================================================================== --- box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h 2015-06-10 19:46:17 UTC (rev 3579) +++ box/trunk/bin/bbackupd/BackupClientInodeToIDMap.h 2015-06-10 19:46:20 UTC (rev 3580) @@ -40,8 +40,10 @@ void Open(const char *Filename, bool ReadOnly, bool CreateNew); void OpenEmpty(); - void AddToMap(InodeRefType InodeRef, int64_t ObjectID, int64_t InDirectory); - bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const; + void AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath); + bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; void Close(); From subversion at boxbackup.org Wed Jun 10 20:46:22 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 10 Jun 2015 19:46:22 GMT Subject: [Box Backup-commit] COMMIT r3581 - box/trunk/test/bbackupd Message-ID: <201506101946.t5AJkM2O005055@web1.boxbackup.org> Author: chris Date: 2015-06-10 19:46:22 +0000 (Wed, 10 Jun 2015) New Revision: 3581 Modified: box/trunk/test/bbackupd/testbbackupd.cpp Log: Ensure that errored tests are reported as failures, not successes Modified: box/trunk/test/bbackupd/testbbackupd.cpp =================================================================== --- box/trunk/test/bbackupd/testbbackupd.cpp 2015-06-10 19:46:20 UTC (rev 3580) +++ box/trunk/test/bbackupd/testbbackupd.cpp 2015-06-10 19:46:22 UTC (rev 3581) @@ -515,12 +515,13 @@ catch (BoxException &e) \ { \ BOX_NOTICE(__FUNCTION__ << " errored: " << e.what()); \ + failures++; \ bool status = teardown_test_bbackupd(__FUNCTION__, old_failure_count); \ s_test_status[__FUNCTION__] = "ERRORED"; \ return status; \ } - + bool test_basics() { SETUP(); From subversion at boxbackup.org Sun Jun 28 20:36:37 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 28 Jun 2015 19:36:37 GMT Subject: [Box Backup-commit] COMMIT r3582 - in box/trunk: bin/bbackupd lib/backupclient lib/server Message-ID: <201506281936.t5SJabi4052943@web1.boxbackup.org> Author: chris Date: 2015-06-28 19:36:37 +0000 (Sun, 28 Jun 2015) New Revision: 3582 Added: box/trunk/lib/backupclient/ClientException.txt box/trunk/lib/backupclient/Makefile.extra Removed: box/trunk/bin/bbackupd/ClientException.txt box/trunk/bin/bbackupd/Makefile.extra Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp box/trunk/bin/bbackupd/BackupDaemon.h box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp box/trunk/lib/server/Daemon.cpp Log: Refactor to allow non-bbstored store configurations. Make some keys not required in BackupDaemonConfigVerify, and verify them afterwards instead, when the rest of the configuration file has been read (and in future, the store type would be known.) Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.cpp 2015-06-10 19:46:22 UTC (rev 3581) +++ box/trunk/bin/bbackupd/BackupDaemon.cpp 2015-06-28 19:36:37 UTC (rev 3582) @@ -321,7 +321,6 @@ return &BackupDaemonConfigVerify; } -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET // -------------------------------------------------------------------------- // // Function @@ -334,6 +333,36 @@ // -------------------------------------------------------------------------- void BackupDaemon::SetupInInitialProcess() { + const Configuration& config(GetConfiguration()); + + // These keys may or may not be required, depending on the configured + // store type (e.g. not when using Amazon S3 stores), so they can't be + // verified by BackupDaemonConfigVerify. + std::vector requiredKeys; + requiredKeys.push_back("StoreHostname"); + requiredKeys.push_back("CertificateFile"); + requiredKeys.push_back("PrivateKeyFile"); + requiredKeys.push_back("TrustedCAsFile"); + bool missingRequiredKeys = false; + + for(std::vector::const_iterator i = requiredKeys.begin(); + i != requiredKeys.end(); i++) + { + if(!config.KeyExists(*i)) + { + BOX_ERROR("Missing required configuration key: " << *i); + missingRequiredKeys = true; + } + } + + if(missingRequiredKeys) + { + THROW_EXCEPTION_MESSAGE(ClientException, InvalidConfiguration, + "Some required configuration keys are missing in " << + GetConfigFileName()); + } + +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET // Print a warning on this platform if the CommandSocket is used. if(GetConfiguration().KeyExists("CommandSocket")) { @@ -346,8 +375,8 @@ "==============================================================================\n" ); } +#endif } -#endif // -------------------------------------------------------------------------- Modified: box/trunk/bin/bbackupd/BackupDaemon.h =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.h 2015-06-10 19:46:22 UTC (rev 3581) +++ box/trunk/bin/bbackupd/BackupDaemon.h 2015-06-28 19:36:37 UTC (rev 3582) @@ -187,10 +187,8 @@ void DeleteUnusedRootDirEntries(BackupClientContext &rContext); -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET // For warning user about potential security hole virtual void SetupInInitialProcess(); -#endif int UseScriptToSeeIfSyncAllowed(); Deleted: box/trunk/bin/bbackupd/ClientException.txt =================================================================== --- box/trunk/bin/bbackupd/ClientException.txt 2015-06-10 19:46:22 UTC (rev 3581) +++ box/trunk/bin/bbackupd/ClientException.txt 2015-06-28 19:36:37 UTC (rev 3582) @@ -1,11 +0,0 @@ - -# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications. - - -EXCEPTION Client 13 - -Internal 0 -AssertFailed 1 -ClockWentBackwards 2 Invalid (negative) sync period: perhaps your clock is going backwards? -FailedToDeleteStoreObjectInfoFile 3 Failed to delete the StoreObjectInfoFile, backup cannot continue safely. -CorruptStoreObjectInfoFile 4 The store object info file contained an invalid value and is probably corrupt. Try deleting it. Deleted: box/trunk/bin/bbackupd/Makefile.extra =================================================================== --- box/trunk/bin/bbackupd/Makefile.extra 2015-06-10 19:46:22 UTC (rev 3581) +++ box/trunk/bin/bbackupd/Makefile.extra 2015-06-28 19:36:37 UTC (rev 3582) @@ -1,7 +0,0 @@ - -MAKEEXCEPTION = ../../lib/common/makeexception.pl - -# AUTOGEN SEEDING -autogen_ClientException.h autogen_ClientException.cpp: $(MAKEEXCEPTION) ClientException.txt - $(_PERL) $(MAKEEXCEPTION) ClientException.txt - Modified: box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp =================================================================== --- box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp 2015-06-10 19:46:22 UTC (rev 3581) +++ box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp 2015-06-28 19:36:37 UTC (rev 3582) @@ -16,7 +16,7 @@ #include "MemLeakFindOn.h" -static const ConfigurationVerifyKey backuplocationkeys[] = +static const ConfigurationVerifyKey backuplocationkeys[] = { ConfigurationVerifyKey("ExcludeFile", ConfigTest_MultiValueAllowed), ConfigurationVerifyKey("ExcludeFilesRegex", ConfigTest_MultiValueAllowed), @@ -29,7 +29,7 @@ ConfigurationVerifyKey("Path", ConfigTest_Exists | ConfigTest_LastEntry) }; -static const ConfigurationVerify backuplocations[] = +static const ConfigurationVerify backuplocations[] = { { "*", @@ -40,7 +40,7 @@ } }; -static const ConfigurationVerifyKey verifyserverkeys[] = +static const ConfigurationVerifyKey verifyserverkeys[] = { DAEMON_VERIFY_SERVER_KEYS }; @@ -63,7 +63,7 @@ } }; -static const ConfigurationVerifyKey verifyrootkeys[] = +static const ConfigurationVerifyKey verifyrootkeys[] = { ConfigurationVerifyKey("AccountNumber", ConfigTest_Exists | ConfigTest_IsUint32), @@ -92,9 +92,6 @@ ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("DiffingUploadSizeThreshold", ConfigTest_Exists | ConfigTest_IsInt), - ConfigurationVerifyKey("StoreHostname", ConfigTest_Exists), - ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, - BOX_PORT_BBSTORED), ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), // extended log to syslog ConfigurationVerifyKey("ExtendedLogFile", 0), @@ -125,13 +122,16 @@ ConfigurationVerifyKey("TcpNice", ConfigTest_IsBool, false), // optional enable of tcp nice/background mode - ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), - ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), - ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), - ConfigurationVerifyKey("DataDirectory", - ConfigTest_Exists | ConfigTest_LastEntry), + ConfigurationVerifyKey("DataDirectory", ConfigTest_Exists), + // These values are only required for bbstored stores: + ConfigurationVerifyKey("StoreHostname", 0), + ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, + BOX_PORT_BBSTORED), + ConfigurationVerifyKey("CertificateFile", 0), + ConfigurationVerifyKey("PrivateKeyFile", 0), + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_LastEntry), }; const ConfigurationVerify BackupDaemonConfigVerify = Copied: box/trunk/lib/backupclient/ClientException.txt (from rev 3581, box/trunk/bin/bbackupd/ClientException.txt) =================================================================== --- box/trunk/lib/backupclient/ClientException.txt (rev 0) +++ box/trunk/lib/backupclient/ClientException.txt 2015-06-28 19:36:37 UTC (rev 3582) @@ -0,0 +1,12 @@ + +# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications. + + +EXCEPTION Client 13 + +Internal 0 +AssertFailed 1 +ClockWentBackwards 2 Invalid (negative) sync period: perhaps your clock is going backwards? +FailedToDeleteStoreObjectInfoFile 3 Failed to delete the StoreObjectInfoFile, backup cannot continue safely. +CorruptStoreObjectInfoFile 4 The store object info file contained an invalid value and is probably corrupt. Try deleting it. +InvalidConfiguration 5 Some required values are missing or incorrect in the configuration file. Copied: box/trunk/lib/backupclient/Makefile.extra (from rev 3581, box/trunk/bin/bbackupd/Makefile.extra) =================================================================== --- box/trunk/lib/backupclient/Makefile.extra (rev 0) +++ box/trunk/lib/backupclient/Makefile.extra 2015-06-28 19:36:37 UTC (rev 3582) @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_ClientException.h autogen_ClientException.cpp: $(MAKEEXCEPTION) ClientException.txt + $(_PERL) $(MAKEEXCEPTION) ClientException.txt + Modified: box/trunk/lib/server/Daemon.cpp =================================================================== --- box/trunk/lib/server/Daemon.cpp 2015-06-10 19:46:22 UTC (rev 3581) +++ box/trunk/lib/server/Daemon.cpp 2015-06-28 19:36:37 UTC (rev 3582) @@ -376,17 +376,17 @@ BOX_ERROR("Failed to load or verify configuration file"); return false; } - + if(!Configure(*apConfig)) { BOX_ERROR("Failed to verify configuration file"); - return false; + return false; } - + // Store configuration mConfigFileName = rConfigFileName; mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); - + return true; } @@ -416,14 +416,14 @@ BOX_ERROR("Configuration errors: " << errors); return false; } - + // Store configuration mapConfiguration = apConf; - + // Let the derived class have a go at setting up stuff // in the initial process SetupInInitialProcess(); - + return true; } From subversion at boxbackup.org Sun Jun 28 20:36:47 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 28 Jun 2015 19:36:47 GMT Subject: [Box Backup-commit] COMMIT r3583 - in box/trunk: . bin/bbstoreaccounts lib/backupstore Message-ID: <201506281936.t5SJalsn052967@web1.boxbackup.org> Author: chris Date: 2015-06-28 19:36:47 +0000 (Sun, 28 Jun 2015) New Revision: 3583 Added: box/trunk/lib/backupstore/BackupAccountControl.cpp box/trunk/lib/backupstore/BackupAccountControl.h Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp box/trunk/lib/backupstore/BackupStoreAccounts.cpp box/trunk/lib/backupstore/BackupStoreAccounts.h box/trunk/modules.txt Log: Refactor store account control to allow other store types. Move common code into a base class, leaving bbstored-specific code. Add skeleton of an S3 store type. Modified: box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp =================================================================== --- box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp 2015-06-28 19:36:37 UTC (rev 3582) +++ box/trunk/bin/bbstoreaccounts/bbstoreaccounts.cpp 2015-06-28 19:36:47 UTC (rev 3583) @@ -177,8 +177,9 @@ } // Decode limits - softlimit = control.SizeStringToBlocks(argv[3], discnum); - hardlimit = control.SizeStringToBlocks(argv[4], discnum); + int blocksize = control.BlockSizeOfDiscSet(discnum); + softlimit = control.SizeStringToBlocks(argv[3], blocksize); + hardlimit = control.SizeStringToBlocks(argv[4], blocksize); control.CheckSoftHardLimits(softlimit, hardlimit); // Create the account... Added: box/trunk/lib/backupstore/BackupAccountControl.cpp =================================================================== --- box/trunk/lib/backupstore/BackupAccountControl.cpp (rev 0) +++ box/trunk/lib/backupstore/BackupAccountControl.cpp 2015-06-28 19:36:47 UTC (rev 3583) @@ -0,0 +1,131 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.cpp +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include + +#include "BackupAccountControl.h" +#include "BackupStoreInfo.h" +#include "Configuration.h" + +#include "MemLeakFindOn.h" + +void BackupAccountControl::CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) +{ + if(SoftLimit > HardLimit) + { + BOX_FATAL("Soft limit must be less than the hard limit."); + exit(1); + } + if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) + { + BOX_WARNING("We recommend setting the soft limit below " << + MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " << + HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE) + / 100) << " in this case."); + } +} + +int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSize) +{ + // Get number + char *endptr = (char*)string; + int64_t number = strtol(string, &endptr, 0); + if(endptr == string || number == LONG_MIN || number == LONG_MAX) + { + BOX_FATAL("'" << string << "' is not a valid number."); + exit(1); + } + + // Check units + switch(*endptr) + { + case 'M': + case 'm': + // Units: Mb + return (number * 1024*1024) / blockSize; + break; + + case 'G': + case 'g': + // Units: Gb + return (number * 1024*1024*1024) / blockSize; + break; + + case 'B': + case 'b': + // Units: Blocks + // Easy! Just return the number specified. + return number; + break; + + default: + BOX_FATAL(string << " has an invalid units specifier " + "(use B for blocks, M for MB, G for GB, eg 2GB)"); + exit(1); + break; + } +} + +std::string BackupAccountControl::BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize) +{ + return FormatUsageBar(Blocks, Blocks * BlockSize, MaxBlocks * BlockSize, + mMachineReadableOutput); +} + +int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, + int BlockSize) +{ + // Then print out lots of info + std::cout << FormatUsageLineStart("Account ID", mMachineReadableOutput) << + BOX_FORMAT_ACCOUNT(info.GetAccountID()) << std::endl; + std::cout << FormatUsageLineStart("Account Name", mMachineReadableOutput) << + info.GetAccountName() << std::endl; + std::cout << FormatUsageLineStart("Last object ID", mMachineReadableOutput) << + BOX_FORMAT_OBJECTID(info.GetLastObjectIDUsed()) << std::endl; + std::cout << FormatUsageLineStart("Used", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksUsed(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Current files", + mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInCurrentFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Old files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInOldFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Deleted files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDeletedFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDirectories(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Soft limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksSoftLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Hard limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksHardLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Client store marker", mMachineReadableOutput) << + info.GetClientStoreMarker() << std::endl; + std::cout << FormatUsageLineStart("Current Files", mMachineReadableOutput) << + info.GetNumCurrentFiles() << std::endl; + std::cout << FormatUsageLineStart("Old Files", mMachineReadableOutput) << + info.GetNumOldFiles() << std::endl; + std::cout << FormatUsageLineStart("Deleted Files", mMachineReadableOutput) << + info.GetNumDeletedFiles() << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + info.GetNumDirectories() << std::endl; + std::cout << FormatUsageLineStart("Enabled", mMachineReadableOutput) << + (info.IsAccountEnabled() ? "yes" : "no") << std::endl; + + return 0; +} + Added: box/trunk/lib/backupstore/BackupAccountControl.h =================================================================== --- box/trunk/lib/backupstore/BackupAccountControl.h (rev 0) +++ box/trunk/lib/backupstore/BackupAccountControl.h 2015-06-28 19:36:47 UTC (rev 3583) @@ -0,0 +1,56 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.h +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPACCOUNTCONTROL__H +#define BACKUPACCOUNTCONTROL__H + +#include + +#include "BackupStoreAccountDatabase.h" +#include "S3Client.h" +#include "NamedLock.h" + +class BackupStoreInfo; +class Configuration; + +class BackupAccountControl +{ +protected: + const Configuration& mConfig; + bool mMachineReadableOutput; + +public: + BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false) + : mConfig(config), + mMachineReadableOutput(machineReadableOutput) + { } + void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit); + int64_t SizeStringToBlocks(const char *string, int BlockSize); + std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize); + int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize); +}; + +class S3BackupAccountControl : public BackupAccountControl +{ +public: + S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false) + : BackupAccountControl(config, machineReadableOutput) + { } + std::string GetStoreRootURL(); + int CreateAccount(const std::string& name, int32_t SoftLimit, int32_t HardLimit); + int GetBlockSize() { return 4096; } +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 + +#endif // BACKUPACCOUNTCONTROL__H + Modified: box/trunk/lib/backupstore/BackupStoreAccounts.cpp =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccounts.cpp 2015-06-28 19:36:37 UTC (rev 3582) +++ box/trunk/lib/backupstore/BackupStoreAccounts.cpp 2015-06-28 19:36:47 UTC (rev 3583) @@ -209,28 +209,6 @@ } } -BackupStoreAccountsControl::BackupStoreAccountsControl( - const Configuration& config, bool machineReadableOutput) -: mConfig(config), - mMachineReadableOutput(machineReadableOutput) -{ } - -void BackupStoreAccountsControl::CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) -{ - if(SoftLimit > HardLimit) - { - BOX_FATAL("Soft limit must be less than the hard limit."); - exit(1); - } - if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) - { - BOX_WARNING("We recommend setting the soft limit below " << - MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " << - HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE) - / 100) << " in this case."); - } -} - int BackupStoreAccountsControl::BlockSizeOfDiscSet(int discSetNum) { // Get controller, check disc set number @@ -245,57 +223,6 @@ return controller.GetDiscSet(discSetNum).GetBlockSize(); } -std::string BackupStoreAccountsControl::BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int discSetNum) -{ - return FormatUsageBar(Blocks, Blocks * BlockSizeOfDiscSet(discSetNum), - MaxBlocks * BlockSizeOfDiscSet(discSetNum), - mMachineReadableOutput); -} - -int64_t BackupStoreAccountsControl::SizeStringToBlocks(const char *string, int discSetNum) -{ - // Find block size - int blockSize = BlockSizeOfDiscSet(discSetNum); - - // Get number - char *endptr = (char*)string; - int64_t number = strtol(string, &endptr, 0); - if(endptr == string || number == LONG_MIN || number == LONG_MAX) - { - BOX_FATAL("'" << string << "' is not a valid number."); - exit(1); - } - - // Check units - switch(*endptr) - { - case 'M': - case 'm': - // Units: Mb - return (number * 1024*1024) / blockSize; - break; - - case 'G': - case 'g': - // Units: Gb - return (number * 1024*1024*1024) / blockSize; - break; - - case 'B': - case 'b': - // Units: Blocks - // Easy! Just return the number specified. - return number; - break; - - default: - BOX_FATAL(string << " has an invalid units specifier " - "(use B for blocks, M for MB, G for GB, eg 2GB)"); - exit(1); - break; - } -} - int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr, const char *HardLimitStr) { @@ -316,8 +243,9 @@ discSetNum, false /* Read/Write */)); // Change the limits - int64_t softlimit = SizeStringToBlocks(SoftLimitStr, discSetNum); - int64_t hardlimit = SizeStringToBlocks(HardLimitStr, discSetNum); + int blocksize = BlockSizeOfDiscSet(discSetNum); + int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize); + int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize); CheckSoftHardLimits(softlimit, hardlimit); info->ChangeLimits(softlimit, hardlimit); @@ -377,50 +305,9 @@ // Load it in std::auto_ptr info(BackupStoreInfo::Load(ID, rootDir, discSetNum, true /* ReadOnly */)); - - // Then print out lots of info - std::cout << FormatUsageLineStart("Account ID", mMachineReadableOutput) << - BOX_FORMAT_ACCOUNT(ID) << std::endl; - std::cout << FormatUsageLineStart("Account Name", mMachineReadableOutput) << - info->GetAccountName() << std::endl; - std::cout << FormatUsageLineStart("Last object ID", mMachineReadableOutput) << - BOX_FORMAT_OBJECTID(info->GetLastObjectIDUsed()) << std::endl; - std::cout << FormatUsageLineStart("Used", mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksUsed(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Current files", - mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInCurrentFiles(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Old files", mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInOldFiles(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Deleted files", mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInDeletedFiles(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInDirectories(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Soft limit", mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksSoftLimit(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Hard limit", mMachineReadableOutput) << - BlockSizeToString(info->GetBlocksHardLimit(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Client store marker", mMachineReadableOutput) << - info->GetClientStoreMarker() << std::endl; - std::cout << FormatUsageLineStart("Current Files", mMachineReadableOutput) << - info->GetNumCurrentFiles() << std::endl; - std::cout << FormatUsageLineStart("Old Files", mMachineReadableOutput) << - info->GetNumOldFiles() << std::endl; - std::cout << FormatUsageLineStart("Deleted Files", mMachineReadableOutput) << - info->GetNumDeletedFiles() << std::endl; - std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << - info->GetNumDirectories() << std::endl; - std::cout << FormatUsageLineStart("Enabled", mMachineReadableOutput) << - (info->IsAccountEnabled() ? "yes" : "no") << std::endl; - - return 0; + + return BackupAccountControl::PrintAccountInfo(*info, + BlockSizeOfDiscSet(discSetNum)); } int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled) Modified: box/trunk/lib/backupstore/BackupStoreAccounts.h =================================================================== --- box/trunk/lib/backupstore/BackupStoreAccounts.h 2015-06-28 19:36:37 UTC (rev 3582) +++ box/trunk/lib/backupstore/BackupStoreAccounts.h 2015-06-28 19:36:47 UTC (rev 3583) @@ -13,6 +13,7 @@ #include #include "BackupStoreAccountDatabase.h" +#include "BackupAccountControl.h" #include "NamedLock.h" // -------------------------------------------------------------------------- @@ -54,20 +55,14 @@ class Configuration; class UnixUser; -class BackupStoreAccountsControl +class BackupStoreAccountsControl : public BackupAccountControl { -private: - const Configuration& mConfig; - bool mMachineReadableOutput; - public: BackupStoreAccountsControl(const Configuration& config, - bool machineReadableOutput = false); - - void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit); + bool machineReadableOutput = false) + : BackupAccountControl(config, machineReadableOutput) + { } int BlockSizeOfDiscSet(int discSetNum); - std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int discSetNum); - int64_t SizeStringToBlocks(const char *string, int discSetNum); bool OpenAccount(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut, std::auto_ptr apUser, NamedLock* pLock); int SetLimit(int32_t ID, const char *SoftLimitStr, Modified: box/trunk/modules.txt =================================================================== --- box/trunk/modules.txt 2015-06-28 19:36:37 UTC (rev 3582) +++ box/trunk/modules.txt 2015-06-28 19:36:47 UTC (rev 3583) @@ -25,7 +25,7 @@ # Backup system -lib/backupstore lib/server lib/raidfile lib/crypto lib/compress +lib/backupstore lib/server lib/raidfile lib/crypto lib/compress lib/httpserver lib/backupclient lib/backupstore bin/bbackupobjdump lib/backupclient lib/backupstore From subversion at boxbackup.org Sun Jun 28 20:36:53 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 28 Jun 2015 19:36:53 GMT Subject: [Box Backup-commit] COMMIT r3584 - box/trunk/lib/backupclient Message-ID: <201506281936.t5SJar7M052989@web1.boxbackup.org> Author: chris Date: 2015-06-28 19:36:52 +0000 (Sun, 28 Jun 2015) New Revision: 3584 Modified: box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp Log: Add configuration keys for S3 store type. Modified: box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp =================================================================== --- box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp 2015-06-28 19:36:47 UTC (rev 3583) +++ box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp 2015-06-28 19:36:52 UTC (rev 3584) @@ -45,8 +45,18 @@ DAEMON_VERIFY_SERVER_KEYS }; -static const ConfigurationVerify verifyserver[] = +static const ConfigurationVerifyKey verifys3keys[] = { + // These values are only required for Amazon S3-compatible stores + ConfigurationVerifyKey("BucketName", 0), + ConfigurationVerifyKey("Port", ConfigTest_IsInt, 80), + ConfigurationVerifyKey("BasePath", 0), + ConfigurationVerifyKey("AccessKey", 0), + ConfigurationVerifyKey("SecretKey", ConfigTest_LastEntry) +}; + +static const ConfigurationVerify verifyserver[] = +{ { "Server", 0, @@ -55,6 +65,13 @@ 0 }, { + "S3Store", + 0, + verifys3keys, + 0, + 0 + }, + { "BackupLocations", backuplocations, 0, From subversion at boxbackup.org Sun Jun 28 22:48:31 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 28 Jun 2015 21:48:31 GMT Subject: [Box Backup-commit] COMMIT r3585 - box/trunk/test/bbackupd Message-ID: <201506282148.t5SLmVvf056740@web1.boxbackup.org> Author: chris Date: 2015-06-28 21:48:31 +0000 (Sun, 28 Jun 2015) New Revision: 3585 Modified: box/trunk/test/bbackupd/Makefile.extra Log: Compile fix for test/bbackupd after ClientException move Modified: box/trunk/test/bbackupd/Makefile.extra =================================================================== --- box/trunk/test/bbackupd/Makefile.extra 2015-06-28 19:36:52 UTC (rev 3584) +++ box/trunk/test/bbackupd/Makefile.extra 2015-06-28 21:48:31 UTC (rev 3585) @@ -1,4 +1,4 @@ -link-extra: ../../bin/bbackupd/autogen_ClientException.o \ +link-extra: \ ../../bin/bbackupd/BackupClientContext.o \ ../../bin/bbackupd/BackupClientDeleteList.o \ ../../bin/bbackupd/BackupClientDirectoryRecord.o \