From subversion at boxbackup.org Sun Feb 1 21:24:22 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 1 Feb 2015 21:24:22 GMT Subject: [Box Backup-commit] COMMIT r3522 - box/trunk/bin/bbackupd Message-ID: <201502012124.t11LOMPx041508@wm.boxbackup.org> Author: chris Date: 2015-02-01 21:24:21 +0000 (Sun, 01 Feb 2015) New Revision: 3522 Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h Log: Suppress multiple links warnings for now. Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2015-01-04 19:41:50 UTC (rev 3521) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2015-02-01 21:24:21 UTC (rev 3522) @@ -56,7 +56,6 @@ mSubDirName(rSubDirName), mInitialSyncDone(false), mSyncDone(false), - mSuppressMultipleLinksWarning(false), mpPendingEntries(0) { ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); @@ -472,31 +471,6 @@ return false; } - int type = file_st.st_mode & S_IFMT; - - // ecryptfs reports nlink > 1 for directories - // with contents, but no filesystem supports - // hardlinking directories? so we can ignore - // this if the entry is a directory. - if(file_st.st_nlink != 1 && type == S_IFDIR) - { - BOX_INFO("Ignoring apparent hard link count on directory: " << - filename << ", nlink=" << file_st.st_nlink); - } - else if(file_st.st_nlink > 1) - { - if(!mSuppressMultipleLinksWarning) - { - BOX_WARNING("File is hard linked, this may cause " - "rename tracking to fail and move files " - "incorrectly in your backup! " << filename << - ", nlink=" << file_st.st_nlink << - " (suppressing further warnings)"); - mSuppressMultipleLinksWarning = true; - } - SetErrorWhenReadingFilesystemObject(rParams, filename); - } - BOX_TRACE("Stat entry '" << filename << "' found device/inode " << file_st.st_dev << "/" << file_st.st_ino); @@ -504,6 +478,7 @@ // a different filesystem than their containing directory, thanks to // Toke Hoiland-Jorgensen. + int type = file_st.st_mode & S_IFMT; if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) { if(!(rParams.mrContext.ExcludeDir(filename))) @@ -904,7 +879,7 @@ if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) - { + { // Only interested if it's a file and the latest version if(!isDir && isCurrentVersion) { @@ -915,7 +890,7 @@ // Doesn't exist locally, but does exist on the server. // Therefore we can safely rename it to this new file. - // Get the connection to the server + // Get the connection to the server BackupProtocolCallable &connection(rContext.GetConnection()); // Only do this step if there is room on the server. Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h 2015-01-04 19:41:50 UTC (rev 3521) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h 2015-02-01 21:24:21 UTC (rev 3522) @@ -202,7 +202,6 @@ std::string mSubDirName; bool mInitialSyncDone; bool mSyncDone; - bool mSuppressMultipleLinksWarning; // Checksum of directory contents and attributes, used to detect changes uint8_t mStateChecksum[MD5Digest::DigestLength]; From subversion at boxbackup.org Sun Feb 1 21:24:26 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 1 Feb 2015 21:24:26 GMT Subject: [Box Backup-commit] COMMIT r3523 - box/trunk/bin/bbackupd Message-ID: <201502012124.t11LOQxS041527@wm.boxbackup.org> Author: chris Date: 2015-02-01 21:24:26 +0000 (Sun, 01 Feb 2015) New Revision: 3523 Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp Log: Fix missing snapshot path on subsequent locations. Thanks to Kai Liebenau for finding and fixing this bug. Also use VssFreeSnapshotProperties instead of rolling our own, also thanks to Kai. This has not been compile tested on Windows. Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.cpp 2015-02-01 21:24:21 UTC (rev 3522) +++ box/trunk/bin/bbackupd/BackupDaemon.cpp 2015-02-01 21:24:26 UTC (rev 3523) @@ -1234,16 +1234,6 @@ return WaitForAsync(pAsync, description); } -void FreeSnapshotProp(VSS_SNAPSHOT_PROP *pSnap) -{ - CoTaskMemFree(pSnap->m_pwszSnapshotDeviceObject); - CoTaskMemFree(pSnap->m_pwszOriginalVolumeName); - CoTaskMemFree(pSnap->m_pwszOriginatingMachine); - CoTaskMemFree(pSnap->m_pwszServiceMachine); - CoTaskMemFree(pSnap->m_pwszExposedName); - CoTaskMemFree(pSnap->m_pwszExposedPath); -} - void BackupDaemon::CreateVssBackupComponents() { std::map volumesIncluded; @@ -1449,21 +1439,6 @@ " to snapshot set"); volumesIncluded[path[0]] = newVolumeId; rLocation.mSnapshotVolumeId = newVolumeId; - rLocation.mIsSnapshotCreated = true; - - // If the snapshot path starts with the volume root - // (drive letter), because the path is absolute (as - // it should be), then remove it so that the - // resulting snapshot path can be appended to the - // snapshot device object to make a real path, - // without a spurious drive letter in it. - - if (path.substr(0, volumeRoot.length()) == volumeRoot) - { - path = path.substr(volumeRoot.length()); - } - - rLocation.mSnapshotPath = path; } else { @@ -1478,8 +1453,23 @@ BOX_TRACE("VSS: Skipping already included volume " << volumeRoot << " for backup location " << path); rLocation.mSnapshotVolumeId = i->second; - rLocation.mIsSnapshotCreated = true; } + + rLocation.mIsSnapshotCreated = true; + + // If the snapshot path starts with the volume root + // (drive letter), because the path is absolute (as it + // should be), then remove it so that the resulting + // snapshot path can be appended to the snapshot device + // object to make a real path, without a spurious drive + // letter in it. + + if (path.substr(0, volumeRoot.length()) == volumeRoot) + { + path = path.substr(volumeRoot.length()); + } + + rLocation.mSnapshotPath = path; } else { @@ -1600,7 +1590,7 @@ rLocation.mSnapshotPath = WideStringToString(prop.m_pwszSnapshotDeviceObject) + DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; - FreeSnapshotProp(&prop); + VssFreeSnapshotProperties(&prop); BOX_INFO("VSS: Location " << rLocation.mPath << " using " "snapshot path " << rLocation.mSnapshotPath); @@ -1698,7 +1688,7 @@ } BOX_TRACE("VSS: Snapshot status: " << status); - FreeSnapshotProp(pSnap); + VssFreeSnapshotProperties(pSnap); } } From subversion at boxbackup.org Sun Feb 15 13:06:03 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sun, 15 Feb 2015 13:06:03 GMT Subject: [Box Backup-commit] COMMIT r3524 - box/trunk/test/basicserver Message-ID: <201502151306.t1FD63mG042978@wm.boxbackup.org> Author: chris Date: 2015-02-15 13:06:02 +0000 (Sun, 15 Feb 2015) New Revision: 3524 Modified: box/trunk/test/basicserver/testbasicserver.cpp Log: Fix test failures caused by renaming the test executable, thanks to Achim for reporting. Modified: box/trunk/test/basicserver/testbasicserver.cpp =================================================================== --- box/trunk/test/basicserver/testbasicserver.cpp 2015-02-01 21:24:26 UTC (rev 3523) +++ box/trunk/test/basicserver/testbasicserver.cpp 2015-02-15 13:06:02 UTC (rev 3524) @@ -486,7 +486,7 @@ // Launch a basic server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = "./_test --test-daemon-args="; cmd += test_args; cmd += " srv1 testfiles/srv1.conf"; int pid = LaunchServer(cmd, "testfiles/srv1.pid"); @@ -532,7 +532,7 @@ // Launch a test forking server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = "./_test --test-daemon-args="; cmd += test_args; cmd += " srv2 testfiles/srv2.conf"; int pid = LaunchServer(cmd, "testfiles/srv2.pid"); @@ -602,7 +602,7 @@ // Launch a test SSL server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = "./_test --test-daemon-args="; cmd += test_args; cmd += " srv3 testfiles/srv3.conf"; int pid = LaunchServer(cmd, "testfiles/srv3.pid"); @@ -683,7 +683,7 @@ //protocolserver: // Launch a test protocol handling server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = "./_test --test-daemon-args="; cmd += test_args; cmd += " srv4 testfiles/srv4.conf"; int pid = LaunchServer(cmd, "testfiles/srv4.pid"); From subversion at boxbackup.org Sat Feb 21 18:32:56 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 21 Feb 2015 18:32:56 GMT Subject: [Box Backup-commit] COMMIT r3525 - box/trunk Message-ID: <201502211832.t1LIWu8r088890@wm.boxbackup.org> Author: jamesog Date: 2015-02-21 18:32:55 +0000 (Sat, 21 Feb 2015) New Revision: 3525 Added: box/trunk/.travis.yml Log: Add configuration file for Travis-CI. Added: box/trunk/.travis.yml =================================================================== --- box/trunk/.travis.yml (rev 0) +++ box/trunk/.travis.yml 2015-02-21 18:32:55 UTC (rev 3525) @@ -0,0 +1,10 @@ +language: cpp +compiler: + - gcc + - clang +script: + - ./bootstrap + - ./configure + - make + - ./runtest.pl ALL debug + - ./runtest.pl ALL release From subversion at boxbackup.org Sat Feb 21 18:40:09 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 21 Feb 2015 18:40:09 GMT Subject: [Box Backup-commit] COMMIT r3526 - box/trunk Message-ID: <201502211840.t1LIe9fv088946@wm.boxbackup.org> Author: jamesog Date: 2015-02-21 18:40:09 +0000 (Sat, 21 Feb 2015) New Revision: 3526 Modified: box/trunk/COPYING.txt box/trunk/LICENSE-DUAL.txt box/trunk/LICENSE-GPL.txt box/trunk/LICENSE.txt Log: Update dates. Modified: box/trunk/COPYING.txt =================================================================== --- box/trunk/COPYING.txt 2015-02-21 18:32:55 UTC (rev 3525) +++ box/trunk/COPYING.txt 2015-02-21 18:40:09 UTC (rev 3526) @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. The license of the code was changed on 23-Jan-2010 in order to meet the Modified: box/trunk/LICENSE-DUAL.txt =================================================================== --- box/trunk/LICENSE-DUAL.txt 2015-02-21 18:32:55 UTC (rev 3525) +++ box/trunk/LICENSE-DUAL.txt 2015-02-21 18:40:09 UTC (rev 3526) @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. Note that this project uses mixed licensing. Any file with this license Modified: box/trunk/LICENSE-GPL.txt =================================================================== --- box/trunk/LICENSE-GPL.txt 2015-02-21 18:32:55 UTC (rev 3525) +++ box/trunk/LICENSE-GPL.txt 2015-02-21 18:40:09 UTC (rev 3526) @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. Note that this project uses mixed licensing. Any file with this license Modified: box/trunk/LICENSE.txt =================================================================== --- box/trunk/LICENSE.txt 2015-02-21 18:32:55 UTC (rev 3525) +++ box/trunk/LICENSE.txt 2015-02-21 18:40:09 UTC (rev 3526) @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. The license of the code was changed on 23-Jan-2010 in order to meet the @@ -76,4 +76,4 @@ The dual license text may be found in the file LICENSE-DUAL.txt, or online at: -[https://www.boxbackup.org/svn/box/trunk/LICENSE-DUAL.txt] \ No newline at end of file +[https://www.boxbackup.org/svn/box/trunk/LICENSE-DUAL.txt] From subversion at boxbackup.org Mon Feb 23 21:38:17 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Mon, 23 Feb 2015 21:38:17 GMT Subject: [Box Backup-commit] COMMIT r3527 - box/trunk/test/backupstorepatch Message-ID: <201502232138.t1NLcHFj004443@wm.boxbackup.org> Author: chris Date: 2015-02-23 21:38:16 +0000 (Mon, 23 Feb 2015) New Revision: 3527 Modified: box/trunk/test/backupstorepatch/testbackupstorepatch.cpp Log: Fix misisng timeouts on network operations. They cause a lot of noise in the Travis build logs. Modified: box/trunk/test/backupstorepatch/testbackupstorepatch.cpp =================================================================== --- box/trunk/test/backupstorepatch/testbackupstorepatch.cpp 2015-02-21 18:40:09 UTC (rev 3526) +++ box/trunk/test/backupstorepatch/testbackupstorepatch.cpp 2015-02-23 21:38:16 UTC (rev 3527) @@ -74,6 +74,7 @@ #define NUMBER_FILES ((sizeof(test_files) / sizeof(test_files[0]))) #define FIRST_FILE_SIZE (64*1024+3) #define BUFFER_SIZE (256*1024) +#define SHORT_TIMEOUT 5000 // Chunk of memory to use for copying files, etc static void *buffer = 0; @@ -194,7 +195,7 @@ FileStream out(fnt, O_WRONLY | O_CREAT); // Copy up to the change point - int b = previous.Read(buffer, test_files[f].ChangePoint, IOStream::TimeOutInfinite); + int b = previous.Read(buffer, test_files[f].ChangePoint, SHORT_TIMEOUT); out.Write(buffer, b); // Add new bytes? @@ -209,7 +210,7 @@ previous.Seek(test_files[f].DeleteBytes, IOStream::SeekType_Relative); } // Copy rest of data - b = previous.Read(buffer, BUFFER_SIZE, IOStream::TimeOutInfinite); + b = previous.Read(buffer, BUFFER_SIZE, SHORT_TIMEOUT); out.Write(buffer, b); } } @@ -245,7 +246,7 @@ // Load the directory back in BackupStoreDirectory dir2; FileStream in("testfiles/dir.1"); - dir2.ReadFromStream(in, IOStream::TimeOutInfinite); + dir2.ReadFromStream(in, SHORT_TIMEOUT); // Check entries TEST_THAT(dir2.GetNumberOfEntries() == 4); for(int i = 2; i <= 5; ++i) @@ -273,7 +274,7 @@ { BackupStoreDirectory dir3; FileStream in("testfiles/dir.2"); - dir3.ReadFromStream(in, IOStream::TimeOutInfinite); + dir3.ReadFromStream(in, SHORT_TIMEOUT); dir3.Dump(0, true); for(int i = 2; i <= 5; ++i) { @@ -442,7 +443,7 @@ // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + dir.ReadFromStream(*dirstream, SHORT_TIMEOUT); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; @@ -482,7 +483,7 @@ BackupStoreDirectory dir; { std::auto_ptr dirStream(RaidFileRead::Open(0, "backup/01234567/o01")); - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dir.ReadFromStream(*dirStream, SHORT_TIMEOUT); dir.Dump(0, true); // Check that dependency info is correct @@ -567,7 +568,7 @@ // Get stream std::auto_ptr filestream(protocol.ReceiveStream()); // Get and decode - BackupStoreFile::DecodeFile(*filestream, filename_fetched, IOStream::TimeOutInfinite); + BackupStoreFile::DecodeFile(*filestream, filename_fetched, SHORT_TIMEOUT); } } // Test for identicalness @@ -578,7 +579,7 @@ std::auto_ptr getblockindex(protocol.QueryGetBlockIndexByID(test_files[f].IDOnServer)); TEST_THAT(getblockindex->GetObjectID() == test_files[f].IDOnServer); std::auto_ptr blockIndexStream(protocol.ReceiveStream()); - TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(filename, *blockIndexStream, IOStream::TimeOutInfinite)); + TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(filename, *blockIndexStream, SHORT_TIMEOUT)); } } From subversion at boxbackup.org Mon Feb 23 22:20:17 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Mon, 23 Feb 2015 22:20:17 GMT Subject: [Box Backup-commit] COMMIT r3528 - box/trunk/test/httpserver Message-ID: <201502232220.t1NMKHeg004657@wm.boxbackup.org> Author: chris Date: 2015-02-23 22:20:16 +0000 (Mon, 23 Feb 2015) New Revision: 3528 Modified: box/trunk/test/httpserver/testhttpserver.cpp Log: Fix testhttpserver, broken by renaming test executables. Modified: box/trunk/test/httpserver/testhttpserver.cpp =================================================================== --- box/trunk/test/httpserver/testhttpserver.cpp 2015-02-23 21:38:16 UTC (rev 3527) +++ box/trunk/test/httpserver/testhttpserver.cpp 2015-02-23 22:20:16 UTC (rev 3528) @@ -145,7 +145,7 @@ } // Start the server - int pid = LaunchServer("./test server testfiles/httpserver.conf", "testfiles/httpserver.pid"); + int pid = LaunchServer("./_test server testfiles/httpserver.conf", "testfiles/httpserver.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid <= 0) { @@ -369,7 +369,7 @@ } // Start the S3Simulator server - pid = LaunchServer("./test s3server testfiles/s3simulator.conf", + pid = LaunchServer("./_test s3server testfiles/s3simulator.conf", "testfiles/s3simulator.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid <= 0) From subversion at boxbackup.org Mon Feb 23 22:26:10 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Mon, 23 Feb 2015 22:26:10 GMT Subject: [Box Backup-commit] COMMIT r3529 - box/trunk/test/backupstorefix/testfiles Message-ID: <201502232226.t1NMQATs004707@wm.boxbackup.org> Author: chris Date: 2015-02-23 22:26:10 +0000 (Mon, 23 Feb 2015) New Revision: 3529 Modified: box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in Log: Fix test failure caused by logging fd change. Logging everything to stdout (so that Windows users can redirect it) causes extra output on stdout that confused the Perl script for this test. Modified: box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in =================================================================== --- box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in 2015-02-23 22:20:16 UTC (rev 3528) +++ box/trunk/test/backupstorefix/testfiles/testbackupstorefix.pl.in 2015-02-23 22:26:10 UTC (rev 3529) @@ -105,6 +105,7 @@ print LISTING_COPY; chomp; s/\r//; s/ \[FILENAME NOT ENCRYPTED\]//; + next if /^WARNING: \*\*\*\* BackupStoreFilename encoded with Clear encoding \*\*\*\*/; if(exists $expected{$_}) { delete $expected{$_} @@ -142,6 +143,7 @@ print LISTING_COPY; chomp; s/\[FILENAME NOT ENCRYPTED\]//; + next if /^WARNING: \*\*\*\* BackupStoreFilename encoded with Clear encoding \*\*\*\*/; my ($id,$type,$name) = split / /; $count++; if($name !~ /\Alost\+found0/) From subversion at boxbackup.org Wed Feb 25 18:56:08 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 25 Feb 2015 18:56:08 GMT Subject: [Box Backup-commit] COMMIT r3530 - in box/trunk: lib/common lib/raidfile test/raidfile Message-ID: <201502251856.t1PIu8dN018320@wm.boxbackup.org> Author: chris Date: 2015-02-25 18:56:07 +0000 (Wed, 25 Feb 2015) New Revision: 3530 Modified: box/trunk/lib/common/Logging.cpp box/trunk/lib/common/Logging.h box/trunk/lib/raidfile/RaidFileRead.cpp box/trunk/lib/raidfile/RaidFileRead.h box/trunk/test/raidfile/testraidfile.cpp Log: Reduce test output noise by hiding some error messages in tests. Should make the Travis logs shorter and more readable. Modified: box/trunk/lib/common/Logging.cpp =================================================================== --- box/trunk/lib/common/Logging.cpp 2015-02-23 22:26:10 UTC (rev 3529) +++ box/trunk/lib/common/Logging.cpp 2015-02-25 18:56:07 UTC (rev 3530) @@ -42,8 +42,9 @@ std::string Logging::sContext; Console* Logging::spConsole = NULL; Syslog* Logging::spSyslog = NULL; -Logging Logging::sGlobalLogging; //automatic initialisation +Logging Logging::sGlobalLogging; // automatic initialisation std::string Logging::sProgramName; +const Log::Category Logging::UNCATEGORISED("Uncategorised"); HideSpecificExceptionGuard::SuppressedExceptions_t HideSpecificExceptionGuard::sSuppressedExceptions; @@ -135,8 +136,9 @@ } } -void Logging::Log(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage) +void Logging::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { std::string newMessage; @@ -145,12 +147,13 @@ newMessage += "[" + sContext + "] "; } - newMessage += rMessage; + newMessage += message; for (std::vector::iterator i = sLoggers.begin(); i != sLoggers.end(); i++) { - bool result = (*i)->Log(level, rFile, line, newMessage); + bool result = (*i)->Log(level, file, line, function, category, + newMessage); if (!result) { return; @@ -158,8 +161,9 @@ } } -void Logging::LogToSyslog(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage) +void Logging::LogToSyslog(Log::Level level, const std::string& rFile, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (!sLogToSyslog) { @@ -173,9 +177,9 @@ newMessage += "[" + sContext + "] "; } - newMessage += rMessage; + newMessage += message; - spSyslog->Log(level, rFile, line, newMessage); + spSyslog->Log(level, rFile, line, function, category, newMessage); } void Logging::SetContext(std::string context) @@ -275,8 +279,9 @@ sShowPID = enabled; } -bool Console::Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) +bool Console::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level > GetLevel()) { @@ -333,7 +338,7 @@ buf << "TRACE: "; } - buf << rMessage; + buf << message; #ifdef WIN32 std::string output = buf.str(); @@ -355,8 +360,9 @@ return true; } -bool Syslog::Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) +bool Syslog::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level > GetLevel()) { @@ -397,7 +403,7 @@ msg = "NOTICE: "; } - msg += rMessage; + msg += message; syslog(syslogLevel, "%s", msg.c_str()); @@ -451,8 +457,9 @@ return LOG_LOCAL6; } -bool FileLogger::Log(Log::Level Level, const std::string& rFile, - int line, std::string& rMessage) +bool FileLogger::Log(Log::Level Level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (mLogFile.StreamClosed()) { @@ -499,7 +506,7 @@ buf << "[TRACE] "; } - buf << rMessage << "\n"; + buf << message << "\n"; std::string output = buf.str(); #ifdef WIN32 @@ -703,3 +710,16 @@ " -V Run at maximum verbosity, log everything to console and system\n" " -W Set verbosity to error/warning/notice/info/trace/everything\n"; } + +bool HideCategoryGuard::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) +{ + // Return false if category is in our list, to suppress further + // logging (thus, return true if it's not in our list, i.e. we + // found nothing). + std::list::iterator i = std::find(mCategories.begin(), + mCategories.end(), category); + return i == mCategories.end(); +} + Modified: box/trunk/lib/common/Logging.h =================================================================== --- box/trunk/lib/common/Logging.h 2015-02-23 22:26:10 UTC (rev 3529) +++ box/trunk/lib/common/Logging.h 2015-02-25 18:56:07 UTC (rev 3530) @@ -12,9 +12,11 @@ #include +#include #include #include #include +#include #include #include @@ -24,14 +26,24 @@ { \ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ - Logging::Log(level, __FILE__, __LINE__, _box_log_line.str()); \ + Logging::Log(level, __FILE__, __LINE__, __FUNCTION__, \ + Logging::UNCATEGORISED, _box_log_line.str()); \ } +#define BOX_LOG_CATEGORY(level, category, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, __FUNCTION__, \ + category, _box_log_line.str()); \ +} + #define BOX_SYSLOG(level, stuff) \ { \ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ - Logging::LogToSyslog(level, __FILE__, __LINE__, _box_log_line.str()); \ + Logging::LogToSyslog(level, __FILE__, __LINE__, __FUNCTION__, \ + _box_log_line.str()); \ } #define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) @@ -83,18 +95,19 @@ BOX_FILE_MESSAGE(filename, message)) #ifdef WIN32 + #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ + stuff << ": " << GetErrorMessage(error_number) << \ + " (" << error_number << ")" #define BOX_LOG_WIN_ERROR(stuff) \ - BOX_ERROR(stuff << ": " << GetErrorMessage(GetLastError())) + BOX_ERROR(BOX_WIN_ERRNO_MESSAGE(GetLastError(), stuff)) #define BOX_LOG_WIN_WARNING(stuff) \ - BOX_WARNING(stuff << ": " << GetErrorMessage(GetLastError())) + BOX_WARNING(BOX_WIN_ERRNO_MESSAGE(GetLastError(), stuff)) #define BOX_LOG_WIN_ERROR_NUMBER(stuff, number) \ - BOX_ERROR(stuff << ": " << GetErrorMessage(number)) + BOX_ERROR(BOX_WIN_ERRNO_MESSAGE(number, stuff)) #define BOX_LOG_WIN_WARNING_NUMBER(stuff, number) \ - BOX_WARNING(stuff << ": " << GetErrorMessage(number)) + BOX_WARNING(BOX_WIN_ERRNO_MESSAGE(number, stuff)) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) - #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ - stuff << ": " << GetErrorMessage(error_number) #define THROW_WIN_ERROR_NUMBER(message, error_number, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ BOX_WIN_ERRNO_MESSAGE(error_number, message)) @@ -166,6 +179,18 @@ EVERYTHING, INVALID = -1 }; + + class Category { + private: + std::string mName; + + public: + Category(const std::string& name) + : mName(name) + { } + const std::string& ToString() { return mName; } + bool operator==(const Category& other) { return mName == other.mName; } + }; } // -------------------------------------------------------------------------- @@ -187,8 +212,9 @@ Logger(Log::Level level); virtual ~Logger(); - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) = 0; + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) = 0; void Filter(Log::Level level) { @@ -243,8 +269,9 @@ static std::string sTag; public: - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "Console"; } virtual void SetProgramName(const std::string& rProgramName); @@ -274,8 +301,9 @@ Syslog(); virtual ~Syslog(); - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "Syslog"; } virtual void SetProgramName(const std::string& rProgramName); virtual void SetFacility(int facility); @@ -297,9 +325,13 @@ public: struct Message { + Message(const Log::Category& category) + : mCategory(category) { } Log::Level level; std::string file; int line; + std::string function; + Log::Category mCategory; std::string message; }; @@ -309,15 +341,17 @@ public: virtual ~Capture() { } - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { - Message message; - message.level = level; - message.file = rFile; - message.line = line; - message.message = rMessage; - mMessages.push_back(message); + Message msg(category); + msg.level = level; + msg.file = file; + msg.line = line; + msg.function = function; + msg.message = message; + mMessages.push_back(msg); return true; } virtual const char* GetType() { return "Capture"; } @@ -366,10 +400,12 @@ static void FilterConsole (Log::Level level); static void Add (Logger* pNewLogger); static void Remove (Logger* pOldLogger); - static void Log(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage); - static void LogToSyslog(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage); + static void Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + static void LogToSyslog(Log::Level level, const std::string& rFile, int line, + const std::string& function, const Log::Category& category, + const std::string& message); static void SetContext(std::string context); static void ClearContext(); static Log::Level GetNamedLevel(const std::string& rName); @@ -481,6 +517,8 @@ return (Log::Level) mCurrentLevel; } }; + + static const Log::Category UNCATEGORISED; }; class FileLogger : public Logger @@ -496,8 +534,9 @@ mLogFile(rFileName, O_WRONLY | O_CREAT | (append ? O_APPEND : O_TRUNC)) { } - virtual bool Log(Log::Level Level, const std::string& rFile, - int Line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "FileLogger"; } virtual void SetProgramName(const std::string& rProgramName) { } @@ -546,6 +585,34 @@ static bool IsHidden(int type, int subtype); }; +class HideCategoryGuard : public Logger +{ + private: + std::list mCategories; + HideCategoryGuard(const HideCategoryGuard& other); // no copying + HideCategoryGuard& operator=(const HideCategoryGuard& other); // no assignment + + public: + HideCategoryGuard(const Log::Category& category) + { + mCategories.push_back(category); + Logging::Add(this); + } + ~HideCategoryGuard() + { + Logging::Remove(this); + } + void Add(const Log::Category& category) + { + mCategories.push_back(category); + } + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + virtual const char* GetType() { return "HideCategoryGuard"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + std::string PrintEscapedBinaryData(const std::string& rInput); #endif // LOGGING__H Modified: box/trunk/lib/raidfile/RaidFileRead.cpp =================================================================== --- box/trunk/lib/raidfile/RaidFileRead.cpp 2015-02-23 22:26:10 UTC (rev 3529) +++ box/trunk/lib/raidfile/RaidFileRead.cpp 2015-02-25 18:56:07 UTC (rev 3530) @@ -47,6 +47,10 @@ // We want to use POSIX fstat() for now, not the emulated one, because it's // difficult to rewrite all this code to use HANDLEs instead of ints. +const RaidFileReadCategory RaidFileRead::OPEN_IN_RECOVERY("OpenInRecovery"); +const RaidFileReadCategory RaidFileRead::IO_ERROR("IoError"); +const RaidFileReadCategory RaidFileRead::RECOVERING_IO_ERROR("RecoverIoError"); + // -------------------------------------------------------------------------- // // Class @@ -548,7 +552,8 @@ // -------------------------------------------------------------------------- void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1) { - BOX_WARNING("Attempting to recover from I/O error: " << mSetNumber << + BOX_LOG_CATEGORY(Log::WARNING, RaidFileRead::RECOVERING_IO_ERROR, + "Attempting to recover from I/O error: " << mSetNumber << " " << mFilename << ", on stripe " << (Stripe1?1:2)); // Close offending file @@ -861,10 +866,10 @@ { if(errno == EIO) { - BOX_ERROR("I/O error when seeking in " << - mSetNumber << " " << mFilename << - " (to " << FilePosition << "), " << - "stripe 1"); + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::IO_ERROR, + "I/O error when seeking in set " << mSetNumber << + ": " << mFilename << " (to " << FilePosition << + "), " << "stripe 1"); // Attempt to recover AttemptToRecoverFromIOError(true /* is stripe 1 */); ASSERT(mStripe1Handle == -1); @@ -881,10 +886,10 @@ { if(errno == EIO) { - BOX_ERROR("I/O error when seeking in " << - mSetNumber << " " << mFilename << - " (to " << FilePosition << "), " << - "stripe 2"); + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::IO_ERROR, + "I/O error when seeking in set " << mSetNumber << + ": " << mFilename << " (to " << FilePosition << + "), " << "stripe 2"); // Attempt to recover AttemptToRecoverFromIOError(false /* is stripe 2 */); ASSERT(mStripe2Handle == -1); @@ -1155,8 +1160,9 @@ bool oktotryagain = true; if(stripe1errno == EIO) { - BOX_ERROR("I/O error on opening " << - SetNumber << " " << Filename << + BOX_LOG_CATEGORY(Log::ERROR, + RaidFileRead::RECOVERING_IO_ERROR, "I/O error " + "on opening " << SetNumber << " " << Filename << " stripe 1, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */); @@ -1172,8 +1178,9 @@ if(stripe2errno == EIO) { - BOX_ERROR("I/O error on opening " << - SetNumber << " " << Filename << + BOX_LOG_CATEGORY(Log::ERROR, + RaidFileRead::RECOVERING_IO_ERROR, "I/O error " + "on opening " << SetNumber << " " << Filename << " stripe 2, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */); @@ -1196,7 +1203,8 @@ if(existance == RaidFileUtil::AsRaidWithMissingReadable) { - BOX_ERROR("Attempting to open RAID file " << SetNumber << + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::OPEN_IN_RECOVERY, + "Attempting to open RAID file " << SetNumber << " " << Filename << " in recovery mode (stripe " << ((existingFiles & RaidFileUtil::Stripe1Exists)?1:2) << " present)"); Modified: box/trunk/lib/raidfile/RaidFileRead.h =================================================================== --- box/trunk/lib/raidfile/RaidFileRead.h 2015-02-23 22:26:10 UTC (rev 3529) +++ box/trunk/lib/raidfile/RaidFileRead.h 2015-02-25 18:56:07 UTC (rev 3530) @@ -16,9 +16,17 @@ #include #include "IOStream.h" +#include "Logging.h" class RaidFileDiscSet; +class RaidFileReadCategory : public Log::Category +{ + public: + RaidFileReadCategory(const std::string& name) + : Log::Category(std::string("RaidFileRead/") + name) + { } +}; // -------------------------------------------------------------------------- // @@ -66,6 +74,10 @@ typedef int64_t FileSizeType; + static const RaidFileReadCategory OPEN_IN_RECOVERY; + static const RaidFileReadCategory IO_ERROR; + static const RaidFileReadCategory RECOVERING_IO_ERROR; + protected: int mSetNumber; std::string mFilename; Modified: box/trunk/test/raidfile/testraidfile.cpp =================================================================== --- box/trunk/test/raidfile/testraidfile.cpp 2015-02-23 22:26:10 UTC (rev 3529) +++ box/trunk/test/raidfile/testraidfile.cpp 2015-02-25 18:56:07 UTC (rev 3530) @@ -216,6 +216,12 @@ // Be nasty, and create some errors for the RAID stuff to recover from... if(TestRAIDProperties) { + HideCategoryGuard hide(RaidFileRead::OPEN_IN_RECOVERY); + hide.Add(RaidFileRead::IO_ERROR); + hide.Add(RaidFileRead::RECOVERING_IO_ERROR); + HideSpecificExceptionGuard hex(RaidFileException::ExceptionType, + RaidFileException::ErrorOpeningFileForRead); + char stripe1fn[256], stripe1fnRename[256]; sprintf(stripe1fn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" DIRECTORY_SEPARATOR "%s.rf", set, startDisc, filename); @@ -273,7 +279,6 @@ DIRECTORY_SEPARATOR ".raidfile-unreadable" DIRECTORY_SEPARATOR "%s", set, (startDisc + 1) % RAID_NUMBER_DISCS, mungefilename); - #ifdef TRF_CAN_INTERCEPT // Test I/O errors on opening From subversion at boxbackup.org Wed Feb 25 20:24:36 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 25 Feb 2015 20:24:36 GMT Subject: [Box Backup-commit] COMMIT r3531 - in box/trunk: lib/common test/common Message-ID: <201502252024.t1PKOacO018755@wm.boxbackup.org> Author: chris Date: 2015-02-25 20:24:36 +0000 (Wed, 25 Feb 2015) New Revision: 3531 Modified: box/trunk/lib/common/Configuration.cpp box/trunk/lib/common/Configuration.h box/trunk/lib/common/ExcludeList.cpp box/trunk/test/common/testcommon.cpp Log: Reduce verbosity of testcommon by hiding some log messages. Modified: box/trunk/lib/common/Configuration.cpp =================================================================== --- box/trunk/lib/common/Configuration.cpp 2015-02-25 18:56:07 UTC (rev 3530) +++ box/trunk/lib/common/Configuration.cpp 2015-02-25 20:24:36 UTC (rev 3531) @@ -34,6 +34,8 @@ static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; static const bool sValueBooleanValue[] = {true, true, false, false}; +const ConfigurationCategory ConfigurationVerify::VERIFY_ERROR("VerifyError"); + ConfigurationVerifyKey::ConfigurationVerifyKey ( std::string name, @@ -212,8 +214,8 @@ if(!rErrorMsg.empty()) { // An error occured, return now - BOX_ERROR("Error in Configuration::LoadInto: " << - rErrorMsg); + BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + "Error in Configuration::LoadInto: " << rErrorMsg); return std::auto_ptr(0); } @@ -222,8 +224,11 @@ { if(!apConfig->Verify(*pVerify, std::string(), rErrorMsg)) { - BOX_ERROR("Error verifying configuration: " << - rErrorMsg); + BOX_LOG_CATEGORY(Log::ERROR, + ConfigurationVerify::VERIFY_ERROR, + "Error verifying configuration: " << + rErrorMsg.substr(0, rErrorMsg.size() > 0 + ? rErrorMsg.size() - 1 : 0)); return std::auto_ptr(0); } } @@ -425,7 +430,8 @@ if(i == mKeys.end()) { - BOX_ERROR("Missing configuration key: " << rKeyName); + BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + "Missing configuration key: " << rKeyName); THROW_EXCEPTION(CommonException, ConfigNoKey) } else Modified: box/trunk/lib/common/Configuration.h =================================================================== --- box/trunk/lib/common/Configuration.h 2015-02-25 18:56:07 UTC (rev 3530) +++ box/trunk/lib/common/Configuration.h 2015-02-25 20:24:36 UTC (rev 3531) @@ -27,6 +27,14 @@ ConfigTest_IsBool = 32 }; +class ConfigurationCategory : public Log::Category +{ + public: + ConfigurationCategory(const std::string& name) + : Log::Category(std::string("Configuration/") + name) + { } +}; + class ConfigurationVerifyKey { public: @@ -75,6 +83,7 @@ const ConfigurationVerifyKey *mpKeys; int Tests; void *TestFunction; // set to zero for now, will implement later + static const ConfigurationCategory VERIFY_ERROR; }; class FdGetLine; Modified: box/trunk/lib/common/ExcludeList.cpp =================================================================== --- box/trunk/lib/common/ExcludeList.cpp 2015-02-25 18:56:07 UTC (rev 3530) +++ box/trunk/lib/common/ExcludeList.cpp 2015-02-25 20:24:36 UTC (rev 3531) @@ -139,9 +139,10 @@ if (entry.size() > 0 && entry[entry.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR) { - BOX_WARNING("Exclude entry ends in path " - "separator, will never match: " - << entry); + BOX_LOG_CATEGORY(Log::WARNING, + ConfigurationVerify::VERIFY_ERROR, + "Exclude entry ends in path separator, " + "will never match: " << entry); } mDefinite.insert(entry); @@ -198,7 +199,9 @@ { char buf[1024]; regerror(errcode, pregex, buf, sizeof(buf)); - BOX_ERROR("Invalid regular expression: " << + BOX_LOG_CATEGORY(Log::ERROR, + ConfigurationVerify::VERIFY_ERROR, + "Invalid regular expression: " << entry << ": " << buf); THROW_EXCEPTION(CommonException, BadRegularExpression) } Modified: box/trunk/test/common/testcommon.cpp =================================================================== --- box/trunk/test/common/testcommon.cpp 2015-02-25 18:56:07 UTC (rev 3530) +++ box/trunk/test/common/testcommon.cpp 2015-02-25 20:24:36 UTC (rev 3531) @@ -160,8 +160,9 @@ bool IsTriggered() { return mTriggered; } void Reset() { mTriggered = false; } - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level == mTargetLevel) { @@ -407,20 +408,18 @@ DIRECTORY_SEPARATOR "fdgetlinetest.txt"); FdGetLine getline(file); - int l = 0; - while(testfilelines[l] != 0) - { - TEST_THAT(!getline.IsEOF()); - std::string line = getline.GetLine(true); - printf("expected |%s| got |%s|\n", testfilelines[l], - line.c_str()); - TEST_LINE(strcmp(testfilelines[l], line.c_str()) == 0, - line); - l++; - } - TEST_THAT(getline.IsEOF()); - TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); + int l = 0; + while(testfilelines[l] != 0) + { + TEST_THAT(!getline.IsEOF()); + std::string line = getline.GetLine(true); + TEST_EQUAL(testfilelines[l], line); + l++; + } + TEST_THAT(getline.IsEOF()); + TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); } + // and again without pre-processing { FileHandleGuard file("testfiles" @@ -430,7 +429,7 @@ TEST_THAT_ABORTONFAIL(file2 != 0); FdGetLine getline(file); char ll[512]; - + while(!feof(file2)) { fgets(ll, sizeof(ll), file2); @@ -440,18 +439,17 @@ e--; } ll[e] = '\0'; - - TEST_THAT(!getline.IsEOF()); - std::string line = getline.GetLine(false); - //printf("expected |%s| got |%s|\n", ll, line.c_str()); - TEST_THAT(strcmp(ll, line.c_str()) == 0); + + TEST_THAT(!getline.IsEOF()); + std::string line = getline.GetLine(false); + TEST_EQUAL(ll, line); } - TEST_THAT(getline.IsEOF()); - TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); + TEST_THAT(getline.IsEOF()); + TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); fclose(file2); } - + // Then the IOStream version of get line, seeing as we're here... { FileStream file("testfiles" DIRECTORY_SEPARATOR @@ -458,22 +456,21 @@ "fdgetlinetest.txt", O_RDONLY); IOStreamGetLine getline(file); - int l = 0; - while(testfilelines[l] != 0) - { - TEST_THAT(!getline.IsEOF()); - std::string line; - while(!getline.GetLine(line, true)) - ; - printf("expected |%s| got |%s|\n", testfilelines[l], - line.c_str()); - TEST_LINE(strcmp(testfilelines[l], line.c_str()) == 0, - line); - l++; - } - TEST_THAT(getline.IsEOF()); - std::string dummy; - TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); + int l = 0; + while(testfilelines[l] != 0) + { + TEST_THAT(!getline.IsEOF()); + std::string line; + while(!getline.GetLine(line, true)) + { + // skip line + } + TEST_EQUAL(testfilelines[l], line); + l++; + } + TEST_THAT(getline.IsEOF()); + std::string dummy; + TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); } // and again without pre-processing { @@ -495,21 +492,20 @@ e--; } ll[e] = '\0'; - - TEST_THAT(!getline.IsEOF()); - std::string line; - while(!getline.GetLine(line, false)) - ; - //printf("expected |%s| got |%s|\n", ll, line.c_str()); - TEST_THAT(strcmp(ll, line.c_str()) == 0); + + TEST_THAT(!getline.IsEOF()); + std::string line; + while(!getline.GetLine(line, false)) + ; + TEST_EQUAL(ll, line); } - TEST_THAT(getline.IsEOF()); - std::string dummy; - TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); + TEST_THAT(getline.IsEOF()); + std::string dummy; + TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); fclose(file2); } - + // Doesn't exist { std::string errMsg; @@ -569,50 +565,69 @@ TEST_THAT(sub1_3.GetKeyValue("terrible") == "absolutely"); } - static const char *file[] = + static const char *file[][2] = { - "testfiles" DIRECTORY_SEPARATOR "config2.txt", + {"testfiles" DIRECTORY_SEPARATOR "config2.txt", + ".TOPlevel (key) is missing."}, // Value missing from root - "testfiles" DIRECTORY_SEPARATOR "config3.txt", + {"testfiles" DIRECTORY_SEPARATOR "config3.txt", + "Unexpected start block in test1"}, // Unexpected { - "testfiles" DIRECTORY_SEPARATOR "config4.txt", + {"testfiles" DIRECTORY_SEPARATOR "config4.txt", + "Root level has close block -- forgot to terminate subblock?"}, // Missing } - "testfiles" DIRECTORY_SEPARATOR "config5.txt", + {"testfiles" DIRECTORY_SEPARATOR "config5.txt", + "Block subconfig2 wasn't started correctly (no '{' on line of it's own)\n" + "Root level has close block -- forgot to terminate subblock?"}, // { expected, but wasn't there - "testfiles" DIRECTORY_SEPARATOR "config6.txt", + {"testfiles" DIRECTORY_SEPARATOR "config6.txt", + "test1.subconfig2.bing (key) multi value not allowed (duplicated key?)."}, // Duplicate key - "testfiles" DIRECTORY_SEPARATOR "config7.txt", + {"testfiles" DIRECTORY_SEPARATOR "config7.txt", + "Invalid configuration key: = invalid thing here!"}, // Invalid key (no name) - "testfiles" DIRECTORY_SEPARATOR "config8.txt", + {"testfiles" DIRECTORY_SEPARATOR "config8.txt", + "File ended without terminating all subblocks"}, // Not all sub blocks terminated - "testfiles" DIRECTORY_SEPARATOR "config9.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9.txt", + "test1.subconfig3.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config9b.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9b.txt", + "test1.subconfig2.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config9c.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9c.txt", + "test1.subconfig2.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config9d.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9d.txt", + "test1.subconfig3.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config10.txt", + {"testfiles" DIRECTORY_SEPARATOR "config10.txt", + "test1.subconfig.carrots (key) is missing."}, // Missing key (in subblock) - "testfiles" DIRECTORY_SEPARATOR "config11.txt", + {"testfiles" DIRECTORY_SEPARATOR "config11.txt", + "test1.subconfig3.NOTEXPECTED (key) is not a known key. Check spelling and placement."}, // Unknown key - "testfiles" DIRECTORY_SEPARATOR "config12.txt", + {"testfiles" DIRECTORY_SEPARATOR "config12.txt", + ".test1.otherthing (block) is missing."}, // Missing block - "testfiles" DIRECTORY_SEPARATOR "config13.txt", + {"testfiles" DIRECTORY_SEPARATOR "config13.txt", + ".test1.* (block) is missing (a block must be present).\n" + ".test1.otherthing (block) is missing."}, // Subconfig (wildcarded) should exist, but missing (ie nothing present) - "testfiles" DIRECTORY_SEPARATOR "config16.txt", + {"testfiles" DIRECTORY_SEPARATOR "config16.txt", + ".BoolTrue1 (key) is not a valid boolean value."}, // bad boolean value - 0 + {NULL, NULL}, }; - for(int l = 0; file[l] != 0; ++l) + for(int l = 0; file[l][0] != 0; ++l) { + HideCategoryGuard hide(ConfigurationVerify::VERIFY_ERROR); std::string errMsg; - std::auto_ptr pconfig(Configuration::LoadAndVerify(file[l], &verify, errMsg)); + std::auto_ptr pconfig(Configuration::LoadAndVerify(file[l][0], &verify, errMsg)); TEST_THAT(pconfig.get() == 0); - TEST_THAT(!errMsg.empty()); - printf("(%s) Error msg is:\n------\n%s------\n", file[l], errMsg.c_str()); + errMsg = errMsg.substr(0, errMsg.size() > 0 ? errMsg.size() - 1 : 0); + TEST_EQUAL_LINE(file[l][1], errMsg, file[l][0]); } // Check that multivalues happen as expected @@ -764,10 +779,13 @@ // Add regex entries #ifdef HAVE_REGEX_SUPPORT + { + HideCategoryGuard hide(ConfigurationVerify::VERIFY_ERROR); elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")); elist.AddRegexEntries(std::string("")); TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[:not_valid")), CommonException, BadRegularExpression); TEST_THAT(elist.SizeOfRegexList() == 3); + } #else TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")), CommonException, RegexNotSupportedOnThisPlatform); TEST_THAT(elist.SizeOfRegexList() == 0); From subversion at boxbackup.org Wed Feb 25 21:20:02 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 25 Feb 2015 21:20:02 GMT Subject: [Box Backup-commit] COMMIT r3532 - box/trunk/lib/common Message-ID: <201502252120.t1PLK2Mu019179@wm.boxbackup.org> Author: chris Date: 2015-02-25 21:20:02 +0000 (Wed, 25 Feb 2015) New Revision: 3532 Modified: box/trunk/lib/common/Logging.h Log: Fix BOX_SYSLOG macro to provide a category. Modified: box/trunk/lib/common/Logging.h =================================================================== --- box/trunk/lib/common/Logging.h 2015-02-25 20:24:36 UTC (rev 3531) +++ box/trunk/lib/common/Logging.h 2015-02-25 21:20:02 UTC (rev 3532) @@ -43,7 +43,7 @@ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ Logging::LogToSyslog(level, __FILE__, __LINE__, __FUNCTION__, \ - _box_log_line.str()); \ + Logging::UNCATEGORISED, _box_log_line.str()); \ } #define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) From subversion at boxbackup.org Wed Feb 25 22:04:10 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Wed, 25 Feb 2015 22:04:10 GMT Subject: [Box Backup-commit] COMMIT r3533 - box/trunk/qdbm Message-ID: <201502252204.t1PM4AT9019605@wm.boxbackup.org> Author: chris Date: 2015-02-25 22:04:09 +0000 (Wed, 25 Feb 2015) New Revision: 3533 Modified: box/trunk/qdbm/configure.in Log: Fix compile with clang on platforms other than FreeBSD. Modified: box/trunk/qdbm/configure.in =================================================================== --- box/trunk/qdbm/configure.in 2015-02-25 21:20:02 UTC (rev 3532) +++ box/trunk/qdbm/configure.in 2015-02-25 22:04:09 UTC (rev 3533) @@ -179,7 +179,7 @@ MYOPTS="-minline-all-stringops" fi -if uname | egrep -iq 'FreeBSD' && cc -v 2>&1 | egrep -iq ' clang '; then +if $CC -v 2>&1 | egrep -iq ' clang '; then MYOPTS="-O1 -fno-omit-frame-pointer" elif uname | egrep -iq 'SunOS' || \ uname | egrep -iq 'BSD' || \ From subversion at boxbackup.org Thu Feb 26 21:03:06 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Thu, 26 Feb 2015 21:03:06 GMT Subject: [Box Backup-commit] COMMIT r3534 - box/trunk/lib/backupstore Message-ID: <201502262103.t1QL36SY027481@wm.boxbackup.org> Author: chris Date: 2015-02-26 21:03:06 +0000 (Thu, 26 Feb 2015) New Revision: 3534 Modified: box/trunk/lib/backupstore/HousekeepStoreAccount.cpp Log: Fix crash when compiled in release mode caused by refcount not decrementing. The call to decrement the refcount was in an ASSERT macro, so compiled out in release mode. Modified: box/trunk/lib/backupstore/HousekeepStoreAccount.cpp =================================================================== --- box/trunk/lib/backupstore/HousekeepStoreAccount.cpp 2015-02-25 22:04:09 UTC (rev 3533) +++ box/trunk/lib/backupstore/HousekeepStoreAccount.cpp 2015-02-26 21:03:06 UTC (rev 3534) @@ -864,7 +864,8 @@ } // Drop reference count by one. Must now be zero, to delete the file. - ASSERT(!mapNewRefs->RemoveReference(ObjectID)) + bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); + ASSERT(!remaining_refs); // Delete from disc BOX_TRACE("Removing unreferenced object " << From subversion at boxbackup.org Fri Feb 27 20:23:05 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Fri, 27 Feb 2015 20:23:05 GMT Subject: [Box Backup-commit] COMMIT r3535 - box/trunk/test/backupstorefix Message-ID: <201502272023.t1RKN5u6034506@wm.boxbackup.org> Author: chris Date: 2015-02-27 20:23:05 +0000 (Fri, 27 Feb 2015) New Revision: 3535 Modified: box/trunk/test/backupstorefix/testbackupstorefix.cpp Log: Fix testbackupstorefix in release mode. It was checking for raidfiles files that have different filenames in release and debug builds, and that aren't even deleted by the test in release builds. Modified: box/trunk/test/backupstorefix/testbackupstorefix.cpp =================================================================== --- box/trunk/test/backupstorefix/testbackupstorefix.cpp 2015-02-26 21:03:06 UTC (rev 3534) +++ box/trunk/test/backupstorefix/testbackupstorefix.cpp 2015-02-27 20:23:05 UTC (rev 3535) @@ -784,6 +784,8 @@ // file, so checking for AsRaid excludes this possibility. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(discSetNum)); + +#ifndef BOX_RELEASE_BUILD // Only if we destroyed these particular files, above. TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( rdiscSet, "backup/01234567/02/01/o00")); TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( @@ -800,6 +802,7 @@ rdiscSet, "backup/01234567/02/01/01/o02")); TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( rdiscSet, "backup/01234567/02/01/01/o03")); +#endif } // ------------------------------------------------------------------------------------------------ From subversion at boxbackup.org Fri Feb 27 21:33:19 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Fri, 27 Feb 2015 21:33:19 GMT Subject: [Box Backup-commit] COMMIT r3536 - box/trunk Message-ID: <201502272133.t1RLXJ0B034855@wm.boxbackup.org> Author: jamesog Date: 2015-02-27 21:33:19 +0000 (Fri, 27 Feb 2015) New Revision: 3536 Modified: box/trunk/.travis.yml Log: Update Travis build configuration. Install LWP to fix httpserver tests. Only attempt to build the master branch. Modified: box/trunk/.travis.yml =================================================================== --- box/trunk/.travis.yml 2015-02-27 20:23:05 UTC (rev 3535) +++ box/trunk/.travis.yml 2015-02-27 21:33:19 UTC (rev 3536) @@ -1,7 +1,13 @@ language: cpp +branches: + only: + - master compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq libwww-perl script: - ./bootstrap - ./configure From subversion at boxbackup.org Fri Feb 27 22:00:26 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Fri, 27 Feb 2015 22:00:26 GMT Subject: [Box Backup-commit] COMMIT r3537 - box/trunk/test/bbackupd Message-ID: <201502272200.t1RM0QCT035005@wm.boxbackup.org> Author: chris Date: 2015-02-27 22:00:25 +0000 (Fri, 27 Feb 2015) New Revision: 3537 Modified: box/trunk/test/bbackupd/testbbackupd.cpp Log: Fix test_ssl_keepalives to not depend on directory listing order. Modified: box/trunk/test/bbackupd/testbbackupd.cpp =================================================================== --- box/trunk/test/bbackupd/testbbackupd.cpp 2015-02-27 21:33:19 UTC (rev 3536) +++ box/trunk/test/bbackupd/testbbackupd.cpp 2015-02-27 22:00:25 UTC (rev 3537) @@ -1708,31 +1708,40 @@ TEST_EQUAL("Receive IsAlive()", reader.GetLine()); TEST_EQUAL("Send GetIsAlive()", reader.GetLine()); TEST_EQUAL("Receive IsAlive()", reader.GetLine()); - TEST_EQUAL("Send ListDirectory(0x6,0xffff,0xc,true)", - reader.GetLine()); // finished reading dir, download to compare - // The following files should be on the server: - // 00000001 -d---- 00002 (root) - // 00000002 -d---- 00002 Test1 - // 00000003 -d---- 00002 Test1/spacetest - // 00000004 f----- 00002 Test1/spacetest/f1 - // 00000005 f----- 00002 Test1/spacetest/f2 - // 00000006 -d---- 00002 Test1/spacetest/d1 - // 00000007 f----- 00002 Test1/spacetest/d1/f3 - // 00000008 f----- 00002 Test1/spacetest/d1/f4 - // 00000009 -d---- 00002 Test1/spacetest/d2 - // 0000000a -d---- 00002 Test1/spacetest/d3 - // 0000000b -d---- 00002 Test1/spacetest/d3/d4 - // 0000000c f----- 00002 Test1/spacetest/d3/d4/f5 - // 0000000d -d---- 00002 Test1/spacetest/d6 - // 0000000e -d---- 00002 Test1/spacetest/d7 - // 0000000f f--o-- 00002 Test1/spacetest/f1 - // 00000010 f--o-- 00002 Test1/spacetest/f1 - // 00000011 f----- 00002 Test1/spacetest/f1 - // This is 34 blocks total. - + // Finished reading dir, download to compare. We expect a listing of d1 + // now, but we don't know its directory ID yet, so ask the store for it. std::auto_ptr client = connect_and_login(context, 0 /* read-write */); + int64_t test1_id = GetDirID(*client, "Test1", + BackupProtocolListDirectory::RootDirectory); + int64_t spacetest_id = GetDirID(*client, "spacetest", test1_id); + int64_t d1_id = GetDirID(*client, "d1", spacetest_id); + std::ostringstream expected; + expected << "Send ListDirectory(" << BOX_FORMAT_OBJECTID(d1_id) << + ",0xffff,0xc,true)"; + TEST_EQUAL(expected.str(), reader.GetLine()); + + // The following files should be on the server: + // 00000001 -d---- 00002 (root) + // 00000002 -d---- 00002 Test1 + // 00000003 -d---- 00002 Test1/spacetest + // 00000004 f----- 00002 Test1/spacetest/f1 + // 00000005 f----- 00002 Test1/spacetest/f2 + // 00000006 -d---- 00002 Test1/spacetest/d1 + // 00000007 f----- 00002 Test1/spacetest/d1/f3 + // 00000008 f----- 00002 Test1/spacetest/d1/f4 + // 00000009 -d---- 00002 Test1/spacetest/d2 + // 0000000a -d---- 00002 Test1/spacetest/d3 + // 0000000b -d---- 00002 Test1/spacetest/d3/d4 + // 0000000c f----- 00002 Test1/spacetest/d3/d4/f5 + // 0000000d -d---- 00002 Test1/spacetest/d6 + // 0000000e -d---- 00002 Test1/spacetest/d7 + // 0000000f f--o-- 00002 Test1/spacetest/f1 + // 00000010 f--o-- 00002 Test1/spacetest/f1 + // 00000011 f----- 00002 Test1/spacetest/f1 + // This is 34 blocks total. + TEST_THAT(check_num_files(5, 3, 0, 9)); TEST_THAT(check_num_blocks(*client, 10, 6, 0, 18, 34)); client->QueryFinished(); From subversion at boxbackup.org Fri Feb 27 22:11:45 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Fri, 27 Feb 2015 22:11:45 GMT Subject: [Box Backup-commit] COMMIT r3538 - box/trunk Message-ID: <201502272211.t1RMBjbd035076@wm.boxbackup.org> Author: jamesog Date: 2015-02-27 22:11:45 +0000 (Fri, 27 Feb 2015) New Revision: 3538 Modified: box/trunk/.travis.yml Log: More Travis config updates. Install some more dependencies. Update the build script to be a single line so that it fails appropriately. Modified: box/trunk/.travis.yml =================================================================== --- box/trunk/.travis.yml 2015-02-27 22:00:25 UTC (rev 3537) +++ box/trunk/.travis.yml 2015-02-27 22:11:45 UTC (rev 3538) @@ -7,10 +7,5 @@ - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq libwww-perl -script: - - ./bootstrap - - ./configure - - make - - ./runtest.pl ALL debug - - ./runtest.pl ALL release + - sudo apt-get install -qq libssl-dev zlib1g-dev libreadline-dev libdb-dev xsltproc libwww-perl +script: ./bootstrap && ./configure && make && ./runtest.pl ALL debug && ./runtest.pl ALL release From subversion at boxbackup.org Sat Feb 28 19:41:38 2015 From: subversion at boxbackup.org (subversion at boxbackup.org) Date: Sat, 28 Feb 2015 19:41:38 GMT Subject: [Box Backup-commit] COMMIT r3539 - box/trunk/infrastructure Message-ID: <201502281941.t1SJfc9k041772@wm.boxbackup.org> Author: jamesog Date: 2015-02-28 19:41:37 +0000 (Sat, 28 Feb 2015) New Revision: 3539 Modified: box/trunk/infrastructure/BoxPlatform.pm.in Log: Fix build on OS X >= 10.9, which uses GNU Make. Also tidy up some whitespace to shut vim up. Modified: box/trunk/infrastructure/BoxPlatform.pm.in =================================================================== --- box/trunk/infrastructure/BoxPlatform.pm.in 2015-02-27 22:11:45 UTC (rev 3538) +++ box/trunk/infrastructure/BoxPlatform.pm.in 2015-02-28 19:41:37 UTC (rev 3539) @@ -8,7 +8,7 @@ # which OS are we building under? $target_os = '@target_os@'; $target_windows = 0; - $target_windows = 1 if $target_os =~ m'^mingw32' + $target_windows = 1 if $target_os =~ m'^mingw32' or $target_os eq "winnt"; if ($^O eq "MSWin32" and not -x "/usr/bin/uname") @@ -34,7 +34,7 @@ if ($build_os eq 'Darwin' and $xcode_ver < 4) { - $make_command = 'bsdmake'; + $make_command = 'bsdmake'; $bsd_make = 1; } elsif ($build_os eq 'SunOS') @@ -54,7 +54,8 @@ { $make_command = 'make'; $bsd_make = ($build_os ne 'Linux' && $build_os ne 'CYGWIN' && - $build_os ne "MINGW32" && $build_os ne 'GNU/kFreeBSD'); + $build_os ne "MINGW32" && $build_os ne 'GNU/kFreeBSD' && + $build_os ne 'Darwin'); } # blank extra flags by default @@ -129,7 +130,7 @@ print STDERR "$cpus processors detected, will set make to perform concurrent jobs\n"; $sub_make_options = ' -j '.($cpus + 1); } - + # if it's Darwin, if($build_os eq 'Darwin') {