From subversion Wed Nov 11 19:22:17 2009 From: subversion (subversion at boxbackup.org) Date: Wed, 11 Nov 2009 19:22:17 -0000 Subject: [Box Backup-commit] COMMIT r2583 - box/trunk/infrastructure Message-ID: <20091111192149.01A96325093@www.boxbackup.org> Author: chris Date: 2009-11-11 19:21:48 +0000 (Wed, 11 Nov 2009) New Revision: 2583 Modified: box/trunk/infrastructure/makebuildenv.pl.in Log: Add support for Makefile.pre in build infrastructure. Modified: box/trunk/infrastructure/makebuildenv.pl.in =================================================================== --- box/trunk/infrastructure/makebuildenv.pl.in 2009-09-26 16:51:07 UTC (rev 2582) +++ box/trunk/infrastructure/makebuildenv.pl.in 2009-11-11 19:21:48 UTC (rev 2583) @@ -613,6 +613,12 @@ __E } + + # if there is a Makefile.pre, include it now + if(-e "$mod/Makefile.pre") + { + print MAKE ".include \n\n"; + } # read directory opendir DIR,$mod; @@ -733,7 +739,7 @@ $has_deps = 1; $has_deps = 0 if $target_is_library; - # Depenency stuff + # Dependency stuff my $deps_makeinfo; if($has_deps) { From subversion Wed Nov 11 19:22:58 2009 From: subversion (subversion at boxbackup.org) Date: Wed, 11 Nov 2009 19:22:58 -0000 Subject: [Box Backup-commit] COMMIT r2584 - box/trunk/lib/common Message-ID: <20091111192252.7791E325093@www.boxbackup.org> Author: chris Date: 2009-11-11 19:22:52 +0000 (Wed, 11 Nov 2009) New Revision: 2584 Modified: box/trunk/lib/common/Archive.h Log: Add methods to read and write exact 32-bit and 64-bit types. Modified: box/trunk/lib/common/Archive.h =================================================================== --- box/trunk/lib/common/Archive.h 2009-11-11 19:21:48 UTC (rev 2583) +++ box/trunk/lib/common/Archive.h 2009-11-11 19:22:52 UTC (rev 2584) @@ -45,6 +45,7 @@ { Write((int) Item); } + void WriteExact(uint32_t Item) { Write((int)Item); } void Write(int Item) { int32_t privItem = htonl(Item); @@ -55,6 +56,7 @@ int64_t privItem = box_hton64(Item); mrStream.Write(&privItem, sizeof(privItem)); } + void WriteExact(uint64_t Item) { Write(Item); } void Write(uint64_t Item) { uint64_t privItem = box_hton64(Item); @@ -88,6 +90,7 @@ rItemOut = false; } } + void ReadExact(uint32_t &rItemOut) { Read((int&)rItemOut); } void Read(int &rItemOut) { int32_t privItem; @@ -106,6 +109,7 @@ } rItemOut = box_ntoh64(privItem); } + void ReadExact(uint64_t &rItemOut) { Read(rItemOut); } void Read(uint64_t &rItemOut) { uint64_t privItem; From subversion Wed Nov 11 19:24:22 2009 From: subversion (subversion at boxbackup.org) Date: Wed, 11 Nov 2009 19:24:22 -0000 Subject: [Box Backup-commit] COMMIT r2585 - box/trunk/bin/bbstored Message-ID: <20091111192350.719A2325093@www.boxbackup.org> Author: chris Date: 2009-11-11 19:23:50 +0000 (Wed, 11 Nov 2009) New Revision: 2585 Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.cpp Log: Add method to get the number of refcounts adjusted by the last housekeeping, for use by tests. Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.cpp =================================================================== --- box/trunk/bin/bbstored/HousekeepStoreAccount.cpp 2009-11-11 19:22:52 UTC (rev 2584) +++ box/trunk/bin/bbstored/HousekeepStoreAccount.cpp 2009-11-11 19:23:50 UTC (rev 2585) @@ -60,6 +60,7 @@ mFilesDeleted(0), mEmptyDirectoriesDeleted(0), mSuppressRefCountChangeWarnings(false), + mRefCountsAdjusted(0), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { } @@ -273,6 +274,7 @@ } apReferences->SetRefCount(ObjectID, mNewRefCounts[ObjectID]); + mRefCountsAdjusted++; LastUsedObjectIdOnDisk = ObjectID; continue; } @@ -288,6 +290,7 @@ " to " << mNewRefCounts[ObjectID]); apReferences->SetRefCount(ObjectID, mNewRefCounts[ObjectID]); + mRefCountsAdjusted++; } } @@ -306,6 +309,7 @@ " changed from " << OldRefCount << " to " << NewRefCount << " (not found)"); apReferences->SetRefCount(ObjectID, NewRefCount); + mRefCountsAdjusted++; } } @@ -646,7 +650,8 @@ // // Function // Name: HousekeepStoreAccount::DeleteFiles() -// Purpose: Delete the files targetted for deletion, returning true if the operation was interrupted +// Purpose: Delete the files targeted for deletion, returning +// true if the operation was interrupted // Created: 15/12/03 // // -------------------------------------------------------------------------- @@ -713,9 +718,11 @@ // -------------------------------------------------------------------------- // // Function -// Name: HousekeepStoreAccount::DeleteFile(int64_t, int64_t, BackupStoreDirectory &, const std::string &, int64_t) -// Purpose: Delete a file. Takes the directory already loaded in and the filename, -// for efficiency in both the usage senarios. +// Name: HousekeepStoreAccount::DeleteFile(int64_t, int64_t, +// BackupStoreDirectory &, const std::string &, int64_t) +// Purpose: Delete a file. Takes the directory already loaded +// in and the filename, for efficiency in both the +// usage scenarios. // Created: 15/7/04 // // -------------------------------------------------------------------------- @@ -725,7 +732,7 @@ bool wasDeleted = false; bool wasOldVersion = false; int64_t deletedFileSizeInBlocks = 0; - // A pointer to an object which requires commiting if the directory save goes OK + // A pointer to an object which requires committing if the directory save goes OK std::auto_ptr padjustedEntry; // BLOCK { @@ -813,7 +820,8 @@ MakeObjectFilename(ObjectID, objFilename); std::auto_ptr pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry - padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, objFilenameOlder)); + padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, + objFilenameOlder, mNewRefCounts[ObjectID])); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) @@ -853,7 +861,8 @@ // BLOCK int64_t dirRevisedSize = 0; { - RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename); + RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, + mNewRefCounts[InDirectory]); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); @@ -879,11 +888,14 @@ padjustedEntry.reset(); // delete it now } - // Delete from disc + // Drop reference count by one. If it reaches zero, delete the file. + if (--mNewRefCounts[ObjectID] == 0) { + // Delete from disc std::string objFilename; MakeObjectFilename(ObjectID, objFilename); - RaidFileWrite del(mStoreDiscSet, objFilename); + RaidFileWrite del(mStoreDiscSet, objFilename, + mNewRefCounts[ObjectID]); del.Delete(); } @@ -1015,7 +1027,8 @@ } // Write revised parent directory - RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename); + RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, + mNewRefCounts[containingDir.GetObjectID()]); writeDir.Open(true /* allow overwriting */); containingDir.WriteToStream(writeDir); @@ -1033,9 +1046,12 @@ mBlocksInDirectoriesDelta += adjust; } - // Delete the directory itself + + if (--mNewRefCounts[dir.GetObjectID()] == 0) { - RaidFileWrite del(mStoreDiscSet, dirFilename); + // Delete the directory itself + RaidFileWrite del(mStoreDiscSet, dirFilename, + mNewRefCounts[dir.GetObjectID()]); del.Delete(); } From subversion Wed Nov 11 19:24:56 2009 From: subversion (subversion at boxbackup.org) Date: Wed, 11 Nov 2009 19:24:56 -0000 Subject: [Box Backup-commit] COMMIT r2586 - box/trunk/bin/bbstored Message-ID: <20091111192428.1FB02325093@www.boxbackup.org> Author: chris Date: 2009-11-11 19:24:27 +0000 (Wed, 11 Nov 2009) New Revision: 2586 Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.h Log: Add methods to get the number of refcounts adjusted by the last housekeeping, for testing. Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.h =================================================================== --- box/trunk/bin/bbstored/HousekeepStoreAccount.h 2009-11-11 19:23:50 UTC (rev 2585) +++ box/trunk/bin/bbstored/HousekeepStoreAccount.h 2009-11-11 19:24:27 UTC (rev 2586) @@ -40,8 +40,8 @@ ~HousekeepStoreAccount(); void DoHousekeeping(bool KeepTryingForever = false); + int GetRefCountsAdjusted() { return mRefCountsAdjusted; } - private: // utility functions void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut); @@ -102,6 +102,7 @@ // New reference count list std::vector mNewRefCounts; bool mSuppressRefCountChangeWarnings; + int mRefCountsAdjusted; // Poll frequency int mCountUntilNextInterprocessMsgCheck; From subversion Tue Nov 24 22:32:14 2009 From: subversion (subversion at boxbackup.org) Date: Tue, 24 Nov 2009 22:32:14 +0000 (GMT) Subject: [Box Backup-commit] COMMIT r2587 - box/trunk/lib/server Message-ID: <20091124223214.B5432325093@www.boxbackup.org> Author: chris Date: 2009-11-24 22:32:13 +0000 (Tue, 24 Nov 2009) New Revision: 2587 Modified: box/trunk/lib/server/ServerStream.h Log: Add debugging for child processes terminating normally or abnormally, as Brendon Baumgartner reported symptoms that sound like a bbstored child process crashing, and nothing in the logs indicates what happened to it. Modified: box/trunk/lib/server/ServerStream.h =================================================================== --- box/trunk/lib/server/ServerStream.h 2009-11-11 19:24:27 UTC (rev 2586) +++ box/trunk/lib/server/ServerStream.h 2009-11-24 22:32:13 UTC (rev 2587) @@ -302,16 +302,7 @@ // Clean up child processes (if forking daemon) if(ForkToHandleRequests && !IsSingleProcess()) { - int status = 0; - int p = 0; - do - { - if((p = ::waitpid(0 /* any child in process group */, &status, WNOHANG)) == -1 - && errno != ECHILD && errno != EINTR) - { - THROW_EXCEPTION(ServerException, ServerWaitOnChildError) - } - } while(p > 0); + WaitForChildren(); } #endif // !WIN32 } @@ -326,6 +317,49 @@ DeleteSockets(); } + #ifndef WIN32 // no waitpid() on Windows + void WaitForChildren() + { + int p = 0; + do + { + int status = 0; + p = ::waitpid(0 /* any child in process group */, + &status, WNOHANG); + + if(p == -1 && errno != ECHILD && errno != EINTR) + { + THROW_EXCEPTION(ServerException, + ServerWaitOnChildError) + } + else if(p == 0) + { + // no children exited, will return from + // function + } + else if(WIFEXITED(status)) + { + BOX_INFO("child process " << p << " " + "terminated normally"); + } + else if(WIFSIGNALED(status)) + { + int sig = WTERMSIG(status); + BOX_ERROR("child process " << p << " " + "terminated abnormally with " + "signal " << sig); + } + else + { + BOX_WARNING("something unknown happened " + "to child process " << p << ": " + "status = " << status); + } + } + while(p > 0); + } + #endif + virtual void HandleConnection(StreamType &rStream) { Connection(rStream); From trac Thu Nov 26 11:59:30 2009 From: trac (Box Backup) Date: Thu, 26 Nov 2009 11:59:30 -0000 Subject: [Box Backup-commit] #59: Inconsistent treatment of NotifyScript between Box Backup and Boxi In-Reply-To: <042.5347e6a7b9901488a701fc1db410ce7d@boxbackup.org> References: <042.5347e6a7b9901488a701fc1db410ce7d@boxbackup.org> Message-ID: <051.b4a157b2c628155166f6e4518d9c0fde@boxbackup.org> #59: Inconsistent treatment of NotifyScript between Box Backup and Boxi ----------------------+----------------------------------------------------- Reporter: achim | Owner: ben Type: defect | Status: new Priority: normal | Milestone: Component: bbackupd | Version: 0.11rc2 Keywords: | ----------------------+----------------------------------------------------- Comment(by achim): Fix so that bbackup will accept paths with spaces for NotifyScript: change BackupDaemon.cpp line 2196 from // Script to run std::string script(conf.GetKeyValue("NotifyScript") + ' ' + sEventNames[Event]); to // Script to run std::string script("\"" + conf.GetKeyValue("NotifyScript") + "\"" + ' ' + sEventNames[Event]); -- Ticket URL: Box Backup An open source, completely automatic on-line backup system for UNIX. From subversion Thu Nov 26 21:44:23 2009 From: subversion (subversion at boxbackup.org) Date: Thu, 26 Nov 2009 21:44:23 +0000 (GMT) Subject: [Box Backup-commit] COMMIT r2588 - box/trunk/bin/bbackupd Message-ID: <20091126214423.2552C325093@www.boxbackup.org> Author: chris Date: 2009-11-26 21:44:22 +0000 (Thu, 26 Nov 2009) New Revision: 2588 Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp Log: Add location of config file as an additional parameter to the SyncAllowScript and the NotifyScript, thanks to Achim J. Latz. Modified: box/trunk/bin/bbackupd/BackupDaemon.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupDaemon.cpp 2009-11-24 22:32:13 UTC (rev 2587) +++ box/trunk/bin/bbackupd/BackupDaemon.cpp 2009-11-26 21:44:22 UTC (rev 2588) @@ -1021,8 +1021,11 @@ pid_t pid = 0; try { + std::string script("\"" + conf.GetKeyValue("SyncAllowScript") + + "\" \"" + GetConfigFileName() + "\""); + std::auto_ptr pscript(LocalProcessStream( - conf.GetKeyValue("SyncAllowScript").c_str(), pid)); + script.c_str(), pid)); // Read in the result IOStreamGetLine getLine(*pscript); @@ -1044,17 +1047,16 @@ } catch(ConversionException &e) { - BOX_ERROR("Invalid output " - "from SyncAllowScript '" - << conf.GetKeyValue("SyncAllowScript") - << "': '" << line << "'"); + BOX_ERROR("Invalid output from " + "SyncAllowScript " << + script << ": '" << line << "'"); throw; } BOX_NOTICE("Delaying sync by " << waitInSeconds - << " seconds (SyncAllowScript '" + << " seconds (SyncAllowScript " << conf.GetKeyValue("SyncAllowScript") - << "')"); + << ")"); } } @@ -2193,20 +2195,19 @@ } // Script to run - std::string script(conf.GetKeyValue("NotifyScript") + ' ' + - sEventNames[Event]); + std::string script("\"" + conf.GetKeyValue("NotifyScript") + "\" " + + sEventNames[Event] + " \"" + GetConfigFileName() + "\""); // Log what we're about to do BOX_INFO("About to notify administrator about event " - << sEventNames[Event] << ", running script '" - << script << "'"); + << sEventNames[Event] << ", running script " << script); // Then do it int returnCode = ::system(script.c_str()); if(returnCode != 0) { BOX_WARNING("Notify script returned error code: " << - returnCode << " ('" << script << "')"); + returnCode << " (" << script << ")"); } else if(Event != SysadminNotifier::BackupStart && Event != SysadminNotifier::BackupFinish) From subversion Thu Nov 26 21:51:30 2009 From: subversion (subversion at boxbackup.org) Date: Thu, 26 Nov 2009 21:51:30 +0000 (GMT) Subject: [Box Backup-commit] COMMIT r2589 - box/trunk/bin/bbackupd Message-ID: <20091126215130.862AA325093@www.boxbackup.org> Author: chris Date: 2009-11-26 21:51:30 +0000 (Thu, 26 Nov 2009) New Revision: 2589 Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp Log: Add missing space before file age in not uploading log message. Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp =================================================================== --- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2009-11-26 21:44:22 UTC (rev 2588) +++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp 2009-11-26 21:51:30 UTC (rev 2589) @@ -925,7 +925,7 @@ BOX_TRACE("Upload decision: " << filename << ": will not upload " "(modified too recently: " - "only " << age << "seconds ago)"); + "only " << age << " seconds ago)"); } else {