Karim Sabri / gammalib

Commit e864958c026b2f810f981d9c50c5713b0698995c

Authored by Jürgen Knödlseder
1 parent 5269ad65

Add static GApplication::running() method to track parents and children and add … 

…heartbeat to daemon (#3592)
Inline Side-by-side
Showing 5 changed files with 245 additions and 60 deletions
include/GApplication.hpp
  View file @e864958
... ... @@ -75,7 +75,7 @@ public:
75 75 GApplicationPar& operator[](const std::string& name);
76 76 const GApplicationPar& operator[](const std::string& name) const;
77 77  
78   - // Methods
  78 + // Public methods
79 79 void clear(void);
80 80 GApplication* clone(void) const;
81 81 std::string classname(void) const;
... ... @@ -131,6 +131,18 @@ public:
131 131 // Public members
132 132 GLog log; //!< Application logger
133 133  
  134 +#ifndef SWIG
  135 +protected:
  136 +#endif
  137 + // Returns the number of running instance of this method. The number
  138 + // of running instances has been implement as static method to avoid
  139 + // the static initialization order fiasco of static members; using
  140 + // static methods we follow the "construct on first use idiom")
  141 + static int& running() {
  142 + static int m_running = 0;
  143 + return m_running;
  144 + }
  145 +
134 146 protected:
135 147 // Protected methods
136 148 void init_members(void);
... ...
include/GDaemon.hpp
  View file @e864958
... ... @@ -76,7 +76,9 @@ protected:
76 76 void free_members(void);
77 77 void create_lock_file(void);
78 78 void delete_lock_file(void);
  79 + void write_heartbeat(void);
79 80 GFilename lock_filename(void) const;
  81 + GFilename heartbeat_filename(void) const;
80 82 GFilename statistics_filename(void) const;
81 83 pid_t lock_pid(void) const;
82 84 void update_statistics(void);
... ... @@ -89,10 +91,11 @@ protected:
89 91 void update_daily(GXml& xml, const GCsv& statistics);
90 92  
91 93 // Protected members
92   - pid_t m_pid; //!< Process ID
93   - int m_period; //!< Wake-up period in seconds
94   - GLog m_log; //!< Logger
95   - GChatter m_chatter; //!< Chattiness of logger
  94 + pid_t m_pid; //!< Process ID
  95 + int m_period; //!< Wake-up period in seconds
  96 + int m_heartbeat; //!< Heartbeat period in seconds
  97 + GLog m_log; //!< Logger
  98 + GChatter m_chatter; //!< Chattiness of logger
96 99 };
97 100  
98 101  
... ... @@ -109,7 +112,7 @@ std::string GDaemon::classname(void) const
109 112  
110 113  
111 114 /***********************************************************************//**
112   - * @brief Returns daemon lock filename
  115 + * @brief Returns name of daemon lock file
113 116 *
114 117 * @return Daemon lock filename.
115 118 ***************************************************************************/
... ... @@ -119,4 +122,16 @@ GFilename GDaemon::lock_filename(void) const
119 122 return (gammalib::gamma_filename("daemon.lock"));
120 123 }
121 124  
  125 +
  126 +/***********************************************************************//**
  127 + * @brief Returns name of daemon heartbeat file
  128 + *
  129 + * @return Daemon heartbeat filename.
  130 + ***************************************************************************/
  131 +inline
  132 +GFilename GDaemon::heartbeat_filename(void) const
  133 +{
  134 + return (gammalib::gamma_filename("daemon.heartbeat"));
  135 +}
  136 +
122 137 #endif /* GDAEMON_HPP */
... ...
pyext/GApplication.i
  View file @e864958
... ... @@ -25,6 +25,7 @@
25 25 */
26 26 %{
27 27 /* Put headers and other declarations here that are needed for compilation */
  28 +#define SWIG
28 29 #include "GApplication.hpp"
29 30 %}
30 31  
... ... @@ -88,7 +89,7 @@ public:
88 89 %rename(_stamp) stamp;
89 90 %rename(_log) log;
90 91  
91   - // Methods
  92 + // Public methods
92 93 void clear(void);
93 94 GApplication* clone(void) const;
94 95 std::string classname(void) const;
... ... @@ -121,6 +122,12 @@ public:
121 122  
122 123 // Public members
123 124 GLog log; //!< Application logger
  125 +
  126 + // Make methods private in Python by prepending an underscore
  127 + %rename(_running) running;
  128 +
  129 + // Protected methods
  130 + static int& running();
124 131 };
125 132  
126 133  
... ... @@ -296,6 +303,14 @@ GApplication._log_value = _log_value
296 303 GApplication copy() {
297 304 return (*self);
298 305 }
  306 + void _inc_running(void) {
  307 + self->running()++;
  308 + return;
  309 + }
  310 + void _dec_running(void) {
  311 + self->running()--;
  312 + return;
  313 + }
299 314 %pythoncode {
300 315 def __getstate__(self):
301 316 pars = self.pars()
... ...
src/app/GApplication.cpp
  View file @e864958
... ... @@ -32,6 +32,7 @@
32 32 #include <unistd.h> // access() function
33 33 #include <cstdlib> // exit() function
34 34 #include <cstdio> // std::fopen(), etc. functions
  35 +#include <signal.h> // signal() function
35 36 #include "GApplication.hpp"
36 37 #include "GTools.hpp"
37 38 #include "GFits.hpp"
... ... @@ -47,7 +48,6 @@
47 48 /* __ Constants __________________________________________________________ */
48 49  
49 50 /* __ Globals ____________________________________________________________ */
50   -bool g_is_parent = true; //!< Signals if application is parent or child
51 51  
52 52 /* __ Method name definitions ____________________________________________ */
53 53  
... ... @@ -88,7 +88,7 @@ GApplication::GApplication(void)
88 88 * @param[in] name Application name.
89 89 * @param[in] version Application version.
90 90 *
91   - * Constructs an application from an application @p name and @p version. The
  91 + * Constructs an application from an application @p name and @p version. The
92 92 * constructor will set the parameter filename to "<name>.par" and the log
93 93 * filename to "<name>".log. The parameters will be loaded from the parameter
94 94 * file.
... ... @@ -136,7 +136,7 @@ GApplication::GApplication(const std::string&amp; name, const std::string&amp; version)
136 136 * @param[in] version Application version.
137 137 * @param[in] pars Application parameters.
138 138 *
139   - * Constructs an application from an application @p name and @p version. The
  139 + * Constructs an application from an application @p name and @p version. The
140 140 * constructor will set the parameter filename to "<name>.par" and the log
141 141 * filename to "<name>".log. The parameters will be loaded from the parameter
142 142 * file.
... ... @@ -675,7 +675,7 @@ void GApplication::logFileClose(void)
675 675 * @brief Signal terse logging
676 676 *
677 677 * @return True if at least terse logging is requested.
678   - *
  678 + *
679 679 * The terse level is used for the most crucial application information that
680 680 * should be logged in all cases (chatter >= 1).
681 681 ***************************************************************************/
... ... @@ -693,7 +693,7 @@ bool GApplication::logTerse(void) const
693 693 * @brief Signal normal logging
694 694 *
695 695 * @return True if at least normal logging is requested.
696   - *
  696 + *
697 697 * The normal level is used for standard application information that should
698 698 * be logged in normal operations (chatter >= 2).
699 699 ***************************************************************************/
... ... @@ -711,7 +711,7 @@ bool GApplication::logNormal(void) const
711 711 * @brief Signal explicit logging
712 712 *
713 713 * @return True if at least explicit logging is requested.
714   - *
  714 + *
715 715 * The explicit level is used for detailed application information that
716 716 * should be logged in detailed studies (chatter >= 3).
717 717 ***************************************************************************/
... ... @@ -729,7 +729,7 @@ bool GApplication::logExplicit(void) const
729 729 * @brief Signal verbose logging
730 730 *
731 731 * @return True if verbose logging is requested.
732   - *
  732 + *
733 733 * The verbose level is used for full application information (chatter >= 4).
734 734 ***************************************************************************/
735 735 bool GApplication::logVerbose(void) const
... ... @@ -746,7 +746,7 @@ bool GApplication::logVerbose(void) const
746 746 * @brief Signal debug logging
747 747 *
748 748 * @return True if the debugging mode has been selected.
749   - *
  749 + *
750 750 * The debug level is used for application debugging.
751 751 ***************************************************************************/
752 752 bool GApplication::logDebug(void) const
... ... @@ -763,7 +763,7 @@ bool GApplication::logDebug(void) const
763 763 * @brief Return clobber
764 764 *
765 765 * @return True if the clobber parameter is true.
766   - *
  766 + *
767 767 * The clobber indicates if existing files should be overwritten.
768 768 ***************************************************************************/
769 769 bool GApplication::clobber(void) const
... ... @@ -1165,6 +1165,8 @@ std::string GApplication::print(const GChatter&amp; chatter) const
1165 1165 // Append application name, version and arguments
1166 1166 result.append("\n"+gammalib::parformat("Name")+name());
1167 1167 result.append("\n"+gammalib::parformat("Version")+version());
  1168 + result.append("\n"+gammalib::parformat("Running instances"));
  1169 + result.append(gammalib::str(running()));
1168 1170 for (int i = 0; i < m_args.size(); ++i) {
1169 1171 if (i == 0) {
1170 1172 result.append("\n"+gammalib::parformat("Command")+m_args[i]);
... ... @@ -1291,19 +1293,18 @@ void GApplication::free_members(void)
1291 1293 /***********************************************************************//**
1292 1294 * @brief Set statistics according to parent child status
1293 1295 *
1294   - * Makes sure than application statistics are only written for the parent
1295   - * which is the first application that is instantiated. All instances that
1296   - * are subsequenty instantiated are considered as children, and no statistics
1297   - * will be written for them since their statistics are included in those of
1298   - * the parent.
  1296 + * Makes sure that application statistics are only written for the parent
  1297 + * which is the first running instances. If this instance signals that an
  1298 + * instance is already running, this instance is a child process that was
  1299 + * started by another running instance.
1299 1300 ***************************************************************************/
1300 1301 void GApplication::set_statistics(void)
1301 1302 {
1302   - // Enable statistics only for parents
1303   - statistics(g_is_parent);
  1303 + // Determine whether instance is a parent
  1304 + bool parent = (running() < 1);
1304 1305  
1305   - // Make sure that next application is a child
1306   - g_is_parent = false;
  1306 + // Enable statistics only for parents
  1307 + statistics(parent);
1307 1308  
1308 1309 // Return
1309 1310 return;
... ... @@ -1480,9 +1481,26 @@ void GApplication::start_daemon(void) const
1480 1481 // If we have a PID of 0 we are in the child process. In this
1481 1482 // case we create and start the daemon ...
1482 1483 if (pid == 0) {
1483   - GDaemon daemon;
1484   - daemon.start();
1485   - exit(0);
  1484 +
  1485 + // The child process becomes session leader
  1486 + setsid();
  1487 +
  1488 + // Ignore signals
  1489 + signal(SIGCHLD, SIG_IGN); // Ignore if child has stopped
  1490 + signal(SIGHUP, SIG_IGN); // Ignore death of controlling process
  1491 +
  1492 + // Fork for the second time and let the first fork
  1493 + // process terminate
  1494 + pid = fork();
  1495 + if (pid == 0) {
  1496 + GDaemon daemon;
  1497 + daemon.start();
  1498 + exit(EXIT_SUCCESS);
  1499 + }
  1500 + else {
  1501 + exit(EXIT_SUCCESS);
  1502 + }
  1503 +
1486 1504 }
1487 1505  
1488 1506 } // endif: child proces created
... ...
src/support/GDaemon.cpp
  View file @e864958
... ... @@ -32,7 +32,8 @@
32 32 #include <cstdio> // std::fopen()
33 33 #include <unistd.h> // sleep()
34 34 #include <fcntl.h> // for file locking
35   -#include <signal.h> // kill()
  35 +#include <signal.h> // sigaction
  36 +#include <string.h> // memset
36 37 #if defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
37 38 #include <sys/prctl.h> // prctl(), PR_SET_NAME
38 39 #endif
... ... @@ -47,6 +48,7 @@
47 48 /* __ Macros _____________________________________________________________ */
48 49  
49 50 /* __ Coding definitions _________________________________________________ */
  51 +#define G_APPEND_LOG //!< Append output to log file
50 52  
51 53 /* __ Debug definitions __________________________________________________ */
52 54  
... ... @@ -174,12 +176,20 @@ GDaemon* GDaemon::clone(void) const
174 176 ***************************************************************************/
175 177 void GDaemon::start(void)
176 178 {
  179 + // Get process ID
  180 + m_pid = getpid();
  181 +
177 182 // Open logger
  183 + #if defined(G_APPEND_LOG)
  184 + m_log.open(gammalib::gamma_filename("daemon.log"), false);
  185 + #else
178 186 m_log.open(gammalib::gamma_filename("daemon.log"), true);
  187 + #endif
179 188 m_log.chatter((GChatter)m_chatter);
180 189 m_log.date(true);
181 190  
182 191 // Log start up of daemon
  192 + m_log << "[" << (int)(m_pid) << "] ";
183 193 m_log << "GammaLib daemon start up" << std::endl;
184 194  
185 195 // Set daemon name
... ... @@ -190,9 +200,21 @@ void GDaemon::start(void)
190 200 // Create lock file
191 201 create_lock_file();
192 202  
  203 + // Create heartbeat file
  204 + write_heartbeat();
  205 +
193 206 // Main event handling loop
194 207 while (true) {
195 208  
  209 + // If another daemon is running then exit this daemon
  210 + pid_t pid = lock_pid();
  211 + if ((pid != 0) && (pid != m_pid)) {
  212 + m_log << "[" << (int)(m_pid) << "] ";
  213 + m_log << "Another daemon with PID " << (int)(pid) << " is running, ";
  214 + m_log << "exit this daemon" << std::endl;
  215 + break;
  216 + }
  217 +
196 218 // Put all activities into try-catch block
197 219 try {
198 220  
... ... @@ -201,24 +223,38 @@ void GDaemon::start(void)
201 223  
202 224 }
203 225 catch (const std::exception &except) {
204   - m_log << "*** Exception catched by daemon:" << std::endl;
  226 + m_log << "[" << (int)(m_pid) << "] ";
  227 + m_log << "Exception catched by daemon" << std::endl;
205 228 m_log << except.what() << std::endl;
206 229 }
207 230  
208 231 // Force logger flushing
209 232 m_log.flush(true);
210 233  
211   - // Wait for some period
212   - sleep(m_period);
  234 + // Determine number of heartbeat cycles before wakeup
  235 + int ncycle = int(double(m_period) / double(m_heartbeat) + 0.5);
  236 + if (ncycle < 1) {
  237 + ncycle = 1;
  238 + }
  239 +
  240 + // Loop over heartbeat cycles and wait for heartbeats
  241 + for (int i = 0; i < ncycle; ++i) {
  242 + write_heartbeat();
  243 + sleep(m_heartbeat);
  244 + }
213 245  
214 246 } // endwhile: main event loop
215 247  
216 248 // Log termination of event loop
217   - m_log << "Terminated event loop for unknown reason." << std::endl;
  249 + m_log << "[" << (int)(m_pid) << "] ";
  250 + m_log << "Terminated event loop" << std::endl;
218 251  
219 252 // Delete lock file
220 253 delete_lock_file();
221 254  
  255 + // Force logger flushing
  256 + m_log.flush(true);
  257 +
222 258 // Close log file
223 259 m_log.close();
224 260  
... ... @@ -232,28 +268,82 @@ void GDaemon::start(void)
232 268 *
233 269 * @return True if daemon is alive
234 270 *
235   - * Checks if the daemon is alive. The daemon is alive if a lock file exists
236   - * and if the process with the PID that is found in the lock file is alive.
  271 + * Checks if the daemon is alive by comparing the time found in the heartbeat
  272 + * file to the current time. If the heartbeat file is not older than twice
  273 + * the heartbeat period the daemon is considered alive.
237 274 *
238 275 * This method can be used by a client to check whether a daemon is alive.
  276 + * The method does not ealy on any process ID so that it works even over
  277 + * multiple machines accessing the same disk space.
239 278 ***************************************************************************/
240 279 bool GDaemon::alive(void) const
241 280 {
242 281 // Initialise flag
243 282 bool alive = false;
244 283  
245   - // Get process ID in lock file
246   - pid_t pid = lock_pid();
  284 + // Set current time
  285 + GTime now;
  286 + now.now();
247 287  
248   - // If process ID is positive then check if process is still alive
249   - if (pid > 0) {
  288 + // Get heartbeat filename
  289 + GFilename filename = heartbeat_filename();
250 290  
251   - // Check if process is alive
252   - if (0 == kill(pid, 0)) {
253   - alive = true;
254   - }
255   -
256   - } // endif: process ID was positive
  291 + // OpenMP critical zone to listen for heartbeats
  292 + #pragma omp critical(GDaemon_alive)
  293 + {
  294 +
  295 + // Get file lock. Continue only in case of success.
  296 + struct flock lock;
  297 + lock.l_type = F_RDLCK; // Want a read lock
  298 + lock.l_whence = SEEK_SET; // Want beginning of file
  299 + lock.l_start = 0; // No offset, lock entire file ...
  300 + lock.l_len = 0; // ... to the end
  301 + lock.l_pid = getpid(); // Current process ID
  302 + int fd = open(filename.url().c_str(), O_RDONLY);
  303 + if (fd != -1) {
  304 +
  305 + // Lock file
  306 + fcntl(fd, F_SETLKW, &lock);
  307 +
  308 + // Open heartbeat file. Continue only if opening was successful.
  309 + FILE* fptr = fopen(filename.url().c_str(), "r");
  310 + if (fptr != NULL) {
  311 +
  312 + // Allocate line buffer
  313 + const int n = 1000;
  314 + char line[n];
  315 +
  316 + // Read line
  317 + fgets(line, n-1, fptr);
  318 +
  319 + // Close file
  320 + fclose(fptr);
  321 +
  322 + // Extract time. Put this in a try-catch block to catch
  323 + // any ill-conditioned time string
  324 + try {
  325 + GTime heartbeat;
  326 + heartbeat.utc(std::string(line));
  327 + if ((now - heartbeat) < 2.0 * double(m_heartbeat)) {
  328 + alive = true;
  329 + }
  330 + }
  331 + catch (const std::exception &except) {
  332 + ;
  333 + }
  334 +
  335 + } // endif: lock file opened successfully
  336 +
  337 + // Unlock file
  338 + lock.l_type = F_UNLCK;
  339 + fcntl(fd, F_SETLK, &lock);
  340 +
  341 + // Close file
  342 + close(fd);
  343 +
  344 + } // endif: file locking successful
  345 +
  346 + } // end of OMP critial zone
257 347  
258 348 // Return flag
259 349 return alive;
... ... @@ -282,6 +372,8 @@ std::string GDaemon::print(const GChatter&amp; chatter) const
282 372 result.append(gammalib::str((int)(m_pid)));
283 373 result.append("\n"+gammalib::parformat("Wake up period (s)"));
284 374 result.append(gammalib::str(m_period));
  375 + result.append("\n"+gammalib::parformat("Heartbeat period (s)"));
  376 + result.append(gammalib::str(m_heartbeat));
285 377 result.append("\n"+gammalib::parformat("Logger chatter level"));
286 378 result.append(gammalib::str(m_chatter));
287 379  
... ... @@ -304,8 +396,9 @@ std::string GDaemon::print(const GChatter&amp; chatter) const
304 396 void GDaemon::init_members(void)
305 397 {
306 398 // Initialise members
307   - m_pid = 0; //!< No process ID
308   - m_period = 3600; //!< Wake-up daemon every hour
  399 + m_pid = 0; //!< No process ID
  400 + m_period = 3600; //!< Wake-up daemon every hour
  401 + m_heartbeat = 60; //!< One heartbeat per minute
309 402 m_log.clear();
310 403 m_chatter = NORMAL; //!< NORMAL chatter level
311 404  
... ... @@ -322,10 +415,11 @@ void GDaemon::init_members(void)
322 415 void GDaemon::copy_members(const GDaemon& daemon)
323 416 {
324 417 // Copy members
325   - m_pid = daemon.m_pid;
326   - m_period = daemon.m_period;
327   - m_log = daemon.m_log;
328   - m_chatter = daemon.m_chatter;
  418 + m_pid = daemon.m_pid;
  419 + m_period = daemon.m_period;
  420 + m_heartbeat = daemon.m_heartbeat;
  421 + m_log = daemon.m_log;
  422 + m_chatter = daemon.m_chatter;
329 423  
330 424 // Return
331 425 return;
... ... @@ -357,13 +451,8 @@ void GDaemon::create_lock_file(void)
357 451  
358 452 // Open lock file. Continue only if opening was successful.
359 453 FILE* fptr = fopen(lockfile.url().c_str(), "w");
360   -
361   - // Successful?
362 454 if (fptr != NULL) {
363 455  
364   - // Get process ID
365   - m_pid = getpid();
366   -
367 456 // Write process ID into lockfile
368 457 fprintf(fptr, "%d\n", (int)(m_pid));
369 458  
... ... @@ -371,8 +460,9 @@ void GDaemon::create_lock_file(void)
371 460 fclose(fptr);
372 461  
373 462 // Log creation of lock file
  463 + m_log << "[" << (int)(m_pid) << "] ";
374 464 m_log << "Created lock file \"" << lockfile.url();
375   - m_log << "\" for PID " << (int)(m_pid) << std::endl;
  465 + m_log << "\"" << std::endl;
376 466  
377 467 } // endif: Lock file opened successfully
378 468  
... ... @@ -400,8 +490,9 @@ void GDaemon::delete_lock_file(void)
400 490 std::remove(lockfile.url().c_str());
401 491  
402 492 // Log removal of lock file
403   - m_log << "Removal of lock file \"" << lockfile.url();
404   - m_log << "\" for PID " << (int)(m_pid) << std::endl;
  493 + m_log << "[" << (int)(m_pid) << "] ";
  494 + m_log << "Removed lock file \"" << lockfile.url();
  495 + m_log << "\"" << std::endl;
405 496  
406 497 }
407 498  
... ... @@ -411,6 +502,37 @@ void GDaemon::delete_lock_file(void)
411 502  
412 503  
413 504 /***********************************************************************//**
  505 + * @brief Write heartbeat file
  506 + *
  507 + * Creates and updates the heartbeat file that contains the current UTC time.
  508 + ***************************************************************************/
  509 +void GDaemon::write_heartbeat(void)
  510 +{
  511 + // Set current time
  512 + GTime now;
  513 + now.now();
  514 +
  515 + // Get heartbeat filename
  516 + GFilename filename = heartbeat_filename();
  517 +
  518 + // Open heartbeat file. Continue only if opening was successful.
  519 + FILE* fptr = fopen(filename.url().c_str(), "w");
  520 + if (fptr != NULL) {
  521 +
  522 + // Write current time into heartbeat file
  523 + fprintf(fptr, "%s\n", now.utc().c_str());
  524 +
  525 + // Close lockfile
  526 + fclose(fptr);
  527 +
  528 + } // endif: Heartbeat file opened successfully
  529 +
  530 + // Return
  531 + return;
  532 +}
  533 +
  534 +
  535 +/***********************************************************************//**
414 536 * @brief Returns process ID in lock file
415 537 *
416 538 * @return Process ID in lock file.
... ... @@ -518,6 +640,7 @@ void GDaemon::update_statistics(void)
518 640 void GDaemon::update_xml(const GCsv& statistics)
519 641 {
520 642 // Log updating of high-level statistics
  643 + m_log << "[" << (int)(m_pid) << "] ";
521 644 m_log << "Update high-level statistics" << std::endl;
522 645  
523 646 // Set high-level statistics filename
... ... @@ -538,7 +661,8 @@ void GDaemon::update_xml(const GCsv&amp; statistics)
538 661 }
539 662 catch (const std::exception &except) {
540 663 xml.clear();
541   - m_log << "*** Failure occured in loading high-level statistics ";
  664 + m_log << "[" << (int)(m_pid) << "] ";
  665 + m_log << "Failure occured in loading high-level statistics ";
542 666 m_log << "XML file \"" << filename.url() << "\"" << std::endl;
543 667 m_log << except.what() << std::endl;
544 668 }
... ... @@ -609,6 +733,7 @@ void GDaemon::create_xml(const GFilename&amp; filename)
609 733 data->append("daily");
610 734  
611 735 // Log creation of high-level statistics file
  736 + m_log << "[" << (int)(m_pid) << "] ";
612 737 m_log << "Created high-level statistics XML file \"";
613 738 m_log << filename.url() << "\"" << std::endl;
614 739  
... ...