From e864958c026b2f810f981d9c50c5713b0698995c Mon Sep 17 00:00:00 2001 From: Jürgen Knödlseder Date: Tue, 22 Feb 2022 15:37:16 +0100 Subject: [PATCH] Add static GApplication::running() method to track parents and children and add heartbeat to daemon (#3592) --- include/GApplication.hpp | 14 +++++++++++++- include/GDaemon.hpp | 25 ++++++++++++++++++++----- pyext/GApplication.i | 17 ++++++++++++++++- src/app/GApplication.cpp | 60 +++++++++++++++++++++++++++++++++++++++--------------------- src/support/GDaemon.cpp | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------- 5 files changed, 245 insertions(+), 60 deletions(-) diff --git a/include/GApplication.hpp b/include/GApplication.hpp index 4f8602b..76cae12 100644 --- a/include/GApplication.hpp +++ b/include/GApplication.hpp @@ -75,7 +75,7 @@ public: GApplicationPar& operator[](const std::string& name); const GApplicationPar& operator[](const std::string& name) const; - // Methods + // Public methods void clear(void); GApplication* clone(void) const; std::string classname(void) const; @@ -131,6 +131,18 @@ public: // Public members GLog log; //!< Application logger +#ifndef SWIG +protected: +#endif + // Returns the number of running instance of this method. The number + // of running instances has been implement as static method to avoid + // the static initialization order fiasco of static members; using + // static methods we follow the "construct on first use idiom") + static int& running() { + static int m_running = 0; + return m_running; + } + protected: // Protected methods void init_members(void); diff --git a/include/GDaemon.hpp b/include/GDaemon.hpp index a52d5c0..388ac69 100644 --- a/include/GDaemon.hpp +++ b/include/GDaemon.hpp @@ -76,7 +76,9 @@ protected: void free_members(void); void create_lock_file(void); void delete_lock_file(void); + void write_heartbeat(void); GFilename lock_filename(void) const; + GFilename heartbeat_filename(void) const; GFilename statistics_filename(void) const; pid_t lock_pid(void) const; void update_statistics(void); @@ -89,10 +91,11 @@ protected: void update_daily(GXml& xml, const GCsv& statistics); // Protected members - pid_t m_pid; //!< Process ID - int m_period; //!< Wake-up period in seconds - GLog m_log; //!< Logger - GChatter m_chatter; //!< Chattiness of logger + pid_t m_pid; //!< Process ID + int m_period; //!< Wake-up period in seconds + int m_heartbeat; //!< Heartbeat period in seconds + GLog m_log; //!< Logger + GChatter m_chatter; //!< Chattiness of logger }; @@ -109,7 +112,7 @@ std::string GDaemon::classname(void) const /***********************************************************************//** - * @brief Returns daemon lock filename + * @brief Returns name of daemon lock file * * @return Daemon lock filename. ***************************************************************************/ @@ -119,4 +122,16 @@ GFilename GDaemon::lock_filename(void) const return (gammalib::gamma_filename("daemon.lock")); } + +/***********************************************************************//** + * @brief Returns name of daemon heartbeat file + * + * @return Daemon heartbeat filename. + ***************************************************************************/ +inline +GFilename GDaemon::heartbeat_filename(void) const +{ + return (gammalib::gamma_filename("daemon.heartbeat")); +} + #endif /* GDAEMON_HPP */ diff --git a/pyext/GApplication.i b/pyext/GApplication.i index f4a467e..59eb1a8 100644 --- a/pyext/GApplication.i +++ b/pyext/GApplication.i @@ -25,6 +25,7 @@ */ %{ /* Put headers and other declarations here that are needed for compilation */ +#define SWIG #include "GApplication.hpp" %} @@ -88,7 +89,7 @@ public: %rename(_stamp) stamp; %rename(_log) log; - // Methods + // Public methods void clear(void); GApplication* clone(void) const; std::string classname(void) const; @@ -121,6 +122,12 @@ public: // Public members GLog log; //!< Application logger + + // Make methods private in Python by prepending an underscore + %rename(_running) running; + + // Protected methods + static int& running(); }; @@ -296,6 +303,14 @@ GApplication._log_value = _log_value GApplication copy() { return (*self); } + void _inc_running(void) { + self->running()++; + return; + } + void _dec_running(void) { + self->running()--; + return; + } %pythoncode { def __getstate__(self): pars = self.pars() diff --git a/src/app/GApplication.cpp b/src/app/GApplication.cpp index 84d4143..5c906be 100644 --- a/src/app/GApplication.cpp +++ b/src/app/GApplication.cpp @@ -32,6 +32,7 @@ #include // access() function #include // exit() function #include // std::fopen(), etc. functions +#include // signal() function #include "GApplication.hpp" #include "GTools.hpp" #include "GFits.hpp" @@ -47,7 +48,6 @@ /* __ Constants __________________________________________________________ */ /* __ Globals ____________________________________________________________ */ -bool g_is_parent = true; //!< Signals if application is parent or child /* __ Method name definitions ____________________________________________ */ @@ -88,7 +88,7 @@ GApplication::GApplication(void) * @param[in] name Application name. * @param[in] version Application version. * - * Constructs an application from an application @p name and @p version. The + * Constructs an application from an application @p name and @p version. The * constructor will set the parameter filename to ".par" and the log * filename to "".log. The parameters will be loaded from the parameter * file. @@ -136,7 +136,7 @@ GApplication::GApplication(const std::string& name, const std::string& version) * @param[in] version Application version. * @param[in] pars Application parameters. * - * Constructs an application from an application @p name and @p version. The + * Constructs an application from an application @p name and @p version. The * constructor will set the parameter filename to ".par" and the log * filename to "".log. The parameters will be loaded from the parameter * file. @@ -675,7 +675,7 @@ void GApplication::logFileClose(void) * @brief Signal terse logging * * @return True if at least terse logging is requested. - * + * * The terse level is used for the most crucial application information that * should be logged in all cases (chatter >= 1). ***************************************************************************/ @@ -693,7 +693,7 @@ bool GApplication::logTerse(void) const * @brief Signal normal logging * * @return True if at least normal logging is requested. - * + * * The normal level is used for standard application information that should * be logged in normal operations (chatter >= 2). ***************************************************************************/ @@ -711,7 +711,7 @@ bool GApplication::logNormal(void) const * @brief Signal explicit logging * * @return True if at least explicit logging is requested. - * + * * The explicit level is used for detailed application information that * should be logged in detailed studies (chatter >= 3). ***************************************************************************/ @@ -729,7 +729,7 @@ bool GApplication::logExplicit(void) const * @brief Signal verbose logging * * @return True if verbose logging is requested. - * + * * The verbose level is used for full application information (chatter >= 4). ***************************************************************************/ bool GApplication::logVerbose(void) const @@ -746,7 +746,7 @@ bool GApplication::logVerbose(void) const * @brief Signal debug logging * * @return True if the debugging mode has been selected. - * + * * The debug level is used for application debugging. ***************************************************************************/ bool GApplication::logDebug(void) const @@ -763,7 +763,7 @@ bool GApplication::logDebug(void) const * @brief Return clobber * * @return True if the clobber parameter is true. - * + * * The clobber indicates if existing files should be overwritten. ***************************************************************************/ bool GApplication::clobber(void) const @@ -1165,6 +1165,8 @@ std::string GApplication::print(const GChatter& chatter) const // Append application name, version and arguments result.append("\n"+gammalib::parformat("Name")+name()); result.append("\n"+gammalib::parformat("Version")+version()); + result.append("\n"+gammalib::parformat("Running instances")); + result.append(gammalib::str(running())); for (int i = 0; i < m_args.size(); ++i) { if (i == 0) { result.append("\n"+gammalib::parformat("Command")+m_args[i]); @@ -1291,19 +1293,18 @@ void GApplication::free_members(void) /***********************************************************************//** * @brief Set statistics according to parent child status * - * Makes sure than application statistics are only written for the parent - * which is the first application that is instantiated. All instances that - * are subsequenty instantiated are considered as children, and no statistics - * will be written for them since their statistics are included in those of - * the parent. + * Makes sure that application statistics are only written for the parent + * which is the first running instances. If this instance signals that an + * instance is already running, this instance is a child process that was + * started by another running instance. ***************************************************************************/ void GApplication::set_statistics(void) { - // Enable statistics only for parents - statistics(g_is_parent); + // Determine whether instance is a parent + bool parent = (running() < 1); - // Make sure that next application is a child - g_is_parent = false; + // Enable statistics only for parents + statistics(parent); // Return return; @@ -1480,9 +1481,26 @@ void GApplication::start_daemon(void) const // If we have a PID of 0 we are in the child process. In this // case we create and start the daemon ... if (pid == 0) { - GDaemon daemon; - daemon.start(); - exit(0); + + // The child process becomes session leader + setsid(); + + // Ignore signals + signal(SIGCHLD, SIG_IGN); // Ignore if child has stopped + signal(SIGHUP, SIG_IGN); // Ignore death of controlling process + + // Fork for the second time and let the first fork + // process terminate + pid = fork(); + if (pid == 0) { + GDaemon daemon; + daemon.start(); + exit(EXIT_SUCCESS); + } + else { + exit(EXIT_SUCCESS); + } + } } // endif: child proces created diff --git a/src/support/GDaemon.cpp b/src/support/GDaemon.cpp index c3c9eb6..97d8b6b 100644 --- a/src/support/GDaemon.cpp +++ b/src/support/GDaemon.cpp @@ -32,7 +32,8 @@ #include // std::fopen() #include // sleep() #include // for file locking -#include // kill() +#include // sigaction +#include // memset #if defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) #include // prctl(), PR_SET_NAME #endif @@ -47,6 +48,7 @@ /* __ Macros _____________________________________________________________ */ /* __ Coding definitions _________________________________________________ */ +#define G_APPEND_LOG //!< Append output to log file /* __ Debug definitions __________________________________________________ */ @@ -174,12 +176,20 @@ GDaemon* GDaemon::clone(void) const ***************************************************************************/ void GDaemon::start(void) { + // Get process ID + m_pid = getpid(); + // Open logger + #if defined(G_APPEND_LOG) + m_log.open(gammalib::gamma_filename("daemon.log"), false); + #else m_log.open(gammalib::gamma_filename("daemon.log"), true); + #endif m_log.chatter((GChatter)m_chatter); m_log.date(true); // Log start up of daemon + m_log << "[" << (int)(m_pid) << "] "; m_log << "GammaLib daemon start up" << std::endl; // Set daemon name @@ -190,9 +200,21 @@ void GDaemon::start(void) // Create lock file create_lock_file(); + // Create heartbeat file + write_heartbeat(); + // Main event handling loop while (true) { + // If another daemon is running then exit this daemon + pid_t pid = lock_pid(); + if ((pid != 0) && (pid != m_pid)) { + m_log << "[" << (int)(m_pid) << "] "; + m_log << "Another daemon with PID " << (int)(pid) << " is running, "; + m_log << "exit this daemon" << std::endl; + break; + } + // Put all activities into try-catch block try { @@ -201,24 +223,38 @@ void GDaemon::start(void) } catch (const std::exception &except) { - m_log << "*** Exception catched by daemon:" << std::endl; + m_log << "[" << (int)(m_pid) << "] "; + m_log << "Exception catched by daemon" << std::endl; m_log << except.what() << std::endl; } // Force logger flushing m_log.flush(true); - // Wait for some period - sleep(m_period); + // Determine number of heartbeat cycles before wakeup + int ncycle = int(double(m_period) / double(m_heartbeat) + 0.5); + if (ncycle < 1) { + ncycle = 1; + } + + // Loop over heartbeat cycles and wait for heartbeats + for (int i = 0; i < ncycle; ++i) { + write_heartbeat(); + sleep(m_heartbeat); + } } // endwhile: main event loop // Log termination of event loop - m_log << "Terminated event loop for unknown reason." << std::endl; + m_log << "[" << (int)(m_pid) << "] "; + m_log << "Terminated event loop" << std::endl; // Delete lock file delete_lock_file(); + // Force logger flushing + m_log.flush(true); + // Close log file m_log.close(); @@ -232,28 +268,82 @@ void GDaemon::start(void) * * @return True if daemon is alive * - * Checks if the daemon is alive. The daemon is alive if a lock file exists - * and if the process with the PID that is found in the lock file is alive. + * Checks if the daemon is alive by comparing the time found in the heartbeat + * file to the current time. If the heartbeat file is not older than twice + * the heartbeat period the daemon is considered alive. * * This method can be used by a client to check whether a daemon is alive. + * The method does not ealy on any process ID so that it works even over + * multiple machines accessing the same disk space. ***************************************************************************/ bool GDaemon::alive(void) const { // Initialise flag bool alive = false; - // Get process ID in lock file - pid_t pid = lock_pid(); + // Set current time + GTime now; + now.now(); - // If process ID is positive then check if process is still alive - if (pid > 0) { + // Get heartbeat filename + GFilename filename = heartbeat_filename(); - // Check if process is alive - if (0 == kill(pid, 0)) { - alive = true; - } - - } // endif: process ID was positive + // OpenMP critical zone to listen for heartbeats + #pragma omp critical(GDaemon_alive) + { + + // Get file lock. Continue only in case of success. + struct flock lock; + lock.l_type = F_RDLCK; // Want a read lock + lock.l_whence = SEEK_SET; // Want beginning of file + lock.l_start = 0; // No offset, lock entire file ... + lock.l_len = 0; // ... to the end + lock.l_pid = getpid(); // Current process ID + int fd = open(filename.url().c_str(), O_RDONLY); + if (fd != -1) { + + // Lock file + fcntl(fd, F_SETLKW, &lock); + + // Open heartbeat file. Continue only if opening was successful. + FILE* fptr = fopen(filename.url().c_str(), "r"); + if (fptr != NULL) { + + // Allocate line buffer + const int n = 1000; + char line[n]; + + // Read line + fgets(line, n-1, fptr); + + // Close file + fclose(fptr); + + // Extract time. Put this in a try-catch block to catch + // any ill-conditioned time string + try { + GTime heartbeat; + heartbeat.utc(std::string(line)); + if ((now - heartbeat) < 2.0 * double(m_heartbeat)) { + alive = true; + } + } + catch (const std::exception &except) { + ; + } + + } // endif: lock file opened successfully + + // Unlock file + lock.l_type = F_UNLCK; + fcntl(fd, F_SETLK, &lock); + + // Close file + close(fd); + + } // endif: file locking successful + + } // end of OMP critial zone // Return flag return alive; @@ -282,6 +372,8 @@ std::string GDaemon::print(const GChatter& chatter) const result.append(gammalib::str((int)(m_pid))); result.append("\n"+gammalib::parformat("Wake up period (s)")); result.append(gammalib::str(m_period)); + result.append("\n"+gammalib::parformat("Heartbeat period (s)")); + result.append(gammalib::str(m_heartbeat)); result.append("\n"+gammalib::parformat("Logger chatter level")); result.append(gammalib::str(m_chatter)); @@ -304,8 +396,9 @@ std::string GDaemon::print(const GChatter& chatter) const void GDaemon::init_members(void) { // Initialise members - m_pid = 0; //!< No process ID - m_period = 3600; //!< Wake-up daemon every hour + m_pid = 0; //!< No process ID + m_period = 3600; //!< Wake-up daemon every hour + m_heartbeat = 60; //!< One heartbeat per minute m_log.clear(); m_chatter = NORMAL; //!< NORMAL chatter level @@ -322,10 +415,11 @@ void GDaemon::init_members(void) void GDaemon::copy_members(const GDaemon& daemon) { // Copy members - m_pid = daemon.m_pid; - m_period = daemon.m_period; - m_log = daemon.m_log; - m_chatter = daemon.m_chatter; + m_pid = daemon.m_pid; + m_period = daemon.m_period; + m_heartbeat = daemon.m_heartbeat; + m_log = daemon.m_log; + m_chatter = daemon.m_chatter; // Return return; @@ -357,13 +451,8 @@ void GDaemon::create_lock_file(void) // Open lock file. Continue only if opening was successful. FILE* fptr = fopen(lockfile.url().c_str(), "w"); - - // Successful? if (fptr != NULL) { - // Get process ID - m_pid = getpid(); - // Write process ID into lockfile fprintf(fptr, "%d\n", (int)(m_pid)); @@ -371,8 +460,9 @@ void GDaemon::create_lock_file(void) fclose(fptr); // Log creation of lock file + m_log << "[" << (int)(m_pid) << "] "; m_log << "Created lock file \"" << lockfile.url(); - m_log << "\" for PID " << (int)(m_pid) << std::endl; + m_log << "\"" << std::endl; } // endif: Lock file opened successfully @@ -400,8 +490,9 @@ void GDaemon::delete_lock_file(void) std::remove(lockfile.url().c_str()); // Log removal of lock file - m_log << "Removal of lock file \"" << lockfile.url(); - m_log << "\" for PID " << (int)(m_pid) << std::endl; + m_log << "[" << (int)(m_pid) << "] "; + m_log << "Removed lock file \"" << lockfile.url(); + m_log << "\"" << std::endl; } @@ -411,6 +502,37 @@ void GDaemon::delete_lock_file(void) /***********************************************************************//** + * @brief Write heartbeat file + * + * Creates and updates the heartbeat file that contains the current UTC time. + ***************************************************************************/ +void GDaemon::write_heartbeat(void) +{ + // Set current time + GTime now; + now.now(); + + // Get heartbeat filename + GFilename filename = heartbeat_filename(); + + // Open heartbeat file. Continue only if opening was successful. + FILE* fptr = fopen(filename.url().c_str(), "w"); + if (fptr != NULL) { + + // Write current time into heartbeat file + fprintf(fptr, "%s\n", now.utc().c_str()); + + // Close lockfile + fclose(fptr); + + } // endif: Heartbeat file opened successfully + + // Return + return; +} + + +/***********************************************************************//** * @brief Returns process ID in lock file * * @return Process ID in lock file. @@ -518,6 +640,7 @@ void GDaemon::update_statistics(void) void GDaemon::update_xml(const GCsv& statistics) { // Log updating of high-level statistics + m_log << "[" << (int)(m_pid) << "] "; m_log << "Update high-level statistics" << std::endl; // Set high-level statistics filename @@ -538,7 +661,8 @@ void GDaemon::update_xml(const GCsv& statistics) } catch (const std::exception &except) { xml.clear(); - m_log << "*** Failure occured in loading high-level statistics "; + m_log << "[" << (int)(m_pid) << "] "; + m_log << "Failure occured in loading high-level statistics "; m_log << "XML file \"" << filename.url() << "\"" << std::endl; m_log << except.what() << std::endl; } @@ -609,6 +733,7 @@ void GDaemon::create_xml(const GFilename& filename) data->append("daily"); // Log creation of high-level statistics file + m_log << "[" << (int)(m_pid) << "] "; m_log << "Created high-level statistics XML file \""; m_log << filename.url() << "\"" << std::endl; -- libgit2 0.21.4