Karim Sabri / ctools

Commit 5f703f7f2b477dd1e7e502963bdd3fda97e32d93

Authored by Jürgen Knödlseder
1 parent bd0a89b5

Complement csfootprint script.

Inline Side-by-side
Showing 1 changed file with 239 additions and 55 deletions
cscripts/csfootprint.py
  View file @5f703f7
... ... @@ -18,6 +18,8 @@
18 18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 19 #
20 20 # ==========================================================================
  21 +import os
  22 +import pwd
21 23 import sys
22 24 import gammalib
23 25 import ctools
... ... @@ -217,7 +219,7 @@ class csfootprint(ctools.cscript):
217 219 wall = float(tool.attribute('wall'))
218 220 cpu = float(tool.attribute('cpu'))
219 221 gCO2e = float(tool.attribute('gCO2e'))
220   -
  222 +
221 223 # Update sums
222 224 sum_calls += calls
223 225 sum_wall += wall
... ... @@ -271,17 +273,17 @@ class csfootprint(ctools.cscript):
271 273 """
272 274 # Format according to precision
273 275 if seconds < 60.0:
274   - format = '%.3f seconds' % (seconds)
  276 + format = '%.1f seconds' % (seconds)
275 277 elif seconds < 3600.0:
276   - format = '%.3f minutes' % (seconds/60.0)
  278 + format = '%.1f minutes' % (seconds/60.0)
277 279 else:
278   - format = '%.3f hours' % (seconds/3600.0)
  280 + format = '%.1f hours' % (seconds/3600.0)
279 281  
280 282 # Return format
281 283 return format
282 284  
283 285 # Format carbon footprint
284   - def _format_footprint(self, gCO2e):
  286 + def _format_footprint(self, gCO2e, latex=False):
285 287 """
286 288 Format carbon footprint
287 289  
... ... @@ -289,72 +291,137 @@ class csfootprint(ctools.cscript):
289 291 ----------
290 292 gCO2e : float
291 293 Carbon footprint in gCO2e
  294 + latex : bool, optional
  295 + Use LaTeX formatting
292 296 """
  297 + # Set unit
  298 + if latex:
  299 + unit = r'CO$_2$e'
  300 + else:
  301 + unit = 'CO2e'
  302 +
293 303 # Format according to precision
294 304 if gCO2e < 1000.0:
295   - format = '%.3f g CO2e' % (gCO2e)
  305 + format = '%.1f g %s' % (gCO2e, unit)
296 306 elif gCO2e < 1.0e6:
297   - format = '%.3f kg CO2e' % (gCO2e/1000.0)
  307 + format = '%.1f kg %s' % (gCO2e/1000.0, unit)
298 308 else:
299   - format = '%.3f t CO2e' % (gCO2e/1.0e6)
  309 + format = '%.1f t %s' % (gCO2e/1.0e6, unit)
300 310  
301 311 # Return format
302 312 return format
303 313  
304   - # Log global statistics
305   - def _global_statistics(self, statistics):
  314 + # Get global information
  315 + def _get_global_information(self, statistics, latex=False):
306 316 """
307   - Log global statistics
308   -
  317 + Get global information from statistics
  318 +
309 319 Parameters
310 320 ----------
311 321 statistics : dict
312 322 Statistics dictionary
  323 + latex : bool, optional
  324 + Use LaTeX formatting
313 325 """
314   - # Log header
315   - self._log_header1(gammalib.TERSE, 'Global statistics')
316   -
317 326 # Derive information
318   - tstart = gammalib.GTime(statistics['use_start'])
319   - tstop = gammalib.GTime(statistics['use_stop'])
320   - duration = tstop - tstart
  327 + tstart = gammalib.GTime(statistics['use_start'])
  328 + tstop = gammalib.GTime(statistics['use_stop'])
  329 + duration = tstop - tstart
  330 + cpu_hours = statistics['cpu']/3600.0
321 331 if statistics['cpu'] != 0.0:
322   - ci_cpu = statistics['gCO2e']/(statistics['cpu']/3600.0)
  332 + ci_cpu = statistics['gCO2e']/cpu_hours
323 333 else:
324 334 ci_cpu = 0.0
325 335 if duration != 0.0:
326 336 fp_dur = statistics['gCO2e']/(duration * gammalib.sec2day)
327 337 else:
328 338 fp_dur = 0.0
329   - fp_yr = fp_dur * 365.25
330   - dur_str = self._format_time(duration)
  339 + fp_yr = fp_dur * 365.25
  340 + dur_str = self._format_time(duration)
331 341 if statistics['wall'] != 0.0:
332 342 load_str = '%.1f %%' % (statistics['cpu']/statistics['wall']*100.0)
333 343 else:
334 344 load_str = 'undefined'
335   - ci_cpu_str = self._format_footprint(ci_cpu) + ' / CPU hour'
336   - fp_dur_str = self._format_footprint(fp_dur) + ' / day'
337   - fp_yr_str = self._format_footprint(fp_yr) + ' / year'
  345 + ci_cpu_str = self._format_footprint(ci_cpu, latex) + ' / CPU hour'
  346 + fp_dur_str = self._format_footprint(fp_dur, latex) + ' / day'
  347 + fp_yr_str = self._format_footprint(fp_yr, latex) + ' / year'
338 348 wall_str = self._format_time(statistics['wall'])
339 349 cpu_str = self._format_time(statistics['cpu'])
340   - gCO2e_str = self._format_footprint(statistics['gCO2e'])
  350 + gCO2e_str = self._format_footprint(statistics['gCO2e'], latex)
341 351 date_str = '%s - %s' % (statistics['start'], statistics['stop'])
342 352 use_str = '%s - %s' % (statistics['use_start'], statistics['use_stop'])
343 353  
  354 + # Split footprint into infrastructure and electricity use footprint
  355 + if statistics['cpu'] != 0.0:
  356 + ef_kWh = 108.0
  357 + ef_total = 4.68
  358 + ef_electricity = 2.43
  359 + ef_other = ef_total - ef_electricity
  360 + val_ef = (ci_cpu - ef_other) * ef_kWh / ef_electricity
  361 + gCO2e_infra = ef_other * cpu_hours
  362 + gCO2e_elect = ef_electricity * val_ef / ef_kWh * cpu_hours
  363 + else:
  364 + val_ef = 0.0
  365 + gCO2e_infra = 0.0
  366 + gCO2e_elect = 0.0
  367 + gCO2e_kWh_str = self._format_footprint(val_ef, latex) + ' / kWh'
  368 + gCO2e_infra_str = self._format_footprint(gCO2e_infra, latex)
  369 + gCO2e_elect_str = self._format_footprint(gCO2e_elect, latex)
  370 +
  371 + # Build electricity footprint string
  372 + elect_str = '%s (%s)' % (gCO2e_elect_str, gCO2e_kWh_str)
  373 +
  374 + # Build result directory
  375 + result = {'ci_cpu': ci_cpu_str,
  376 + 'fp_dur': fp_dur_str,
  377 + 'fp_yr': fp_yr_str,
  378 + 'wall': wall_str,
  379 + 'cpu': cpu_str,
  380 + 'gCO2e': gCO2e_str,
  381 + 'date': date_str,
  382 + 'use': use_str,
  383 + 'dur': dur_str,
  384 + 'load': load_str,
  385 + 'gCO2e_kWh': gCO2e_kWh_str,
  386 + 'gCO2e_infra': gCO2e_infra_str,
  387 + 'gCO2e_elect': gCO2e_elect_str,
  388 + 'elect': elect_str}
  389 +
  390 + # Return result
  391 + return result
  392 +
  393 + # Log global statistics
  394 + def _global_statistics(self, statistics):
  395 + """
  396 + Log global statistics
  397 +
  398 + Parameters
  399 + ----------
  400 + statistics : dict
  401 + Statistics dictionary
  402 + """
  403 + # Log header
  404 + self._log_header1(gammalib.TERSE, 'Global statistics')
  405 +
  406 + # Derive information from statistics
  407 + info = self._get_global_information(statistics)
  408 +
344 409 # Log report information
345 410 self._log_value(gammalib.NORMAL, 'Creation date', statistics['creation'])
346 411 self._log_value(gammalib.NORMAL, 'Last statistics update', statistics['modified'])
347   - self._log_value(gammalib.NORMAL, 'Statistics date interval', date_str)
348   - self._log_value(gammalib.NORMAL, 'Used date interval', use_str)
349   - self._log_value(gammalib.NORMAL, 'Duration of used interval', dur_str)
  412 + self._log_value(gammalib.NORMAL, 'Statistics date interval', info['date'])
  413 + self._log_value(gammalib.NORMAL, 'Used date interval', info['use'])
  414 + self._log_value(gammalib.NORMAL, 'Duration of used interval', info['dur'])
350 415 self._log_value(gammalib.NORMAL, 'Total number of ctool runs', statistics['calls'])
351   - self._log_value(gammalib.NORMAL, 'Total wall clock time', wall_str)
352   - self._log_value(gammalib.NORMAL, 'Total CPU time', cpu_str)
353   - self._log_value(gammalib.NORMAL, 'Average CPU load', load_str)
354   - self._log_value(gammalib.NORMAL, 'Total carbon footprint', gCO2e_str)
355   - self._log_value(gammalib.NORMAL, 'Average carbon intensity', ci_cpu_str)
356   - self._log_value(gammalib.NORMAL, 'Average daily footprint', fp_dur_str)
357   - self._log_value(gammalib.NORMAL, 'Expected annual footprint', fp_yr_str)
  416 + self._log_value(gammalib.NORMAL, 'Total wall clock time', info['wall'])
  417 + self._log_value(gammalib.NORMAL, 'Total CPU time', info['cpu'])
  418 + self._log_value(gammalib.NORMAL, 'Average CPU load', info['load'])
  419 + self._log_value(gammalib.NORMAL, 'Total carbon footprint', info['gCO2e'])
  420 + self._log_value(gammalib.NORMAL, ' due to power consumption', info['elect'])
  421 + self._log_value(gammalib.NORMAL, ' due to infrastructure', info['gCO2e_infra'])
  422 + self._log_value(gammalib.NORMAL, 'Average carbon intensity', info['ci_cpu'])
  423 + self._log_value(gammalib.NORMAL, 'Average daily footprint', info['fp_dur'])
  424 + self._log_value(gammalib.NORMAL, 'Expected annual footprint', info['fp_yr'])
358 425  
359 426 # Return
360 427 return
... ... @@ -513,33 +580,86 @@ class csfootprint(ctools.cscript):
513 580 """
514 581 # Optionally use matplotlib to create a figure
515 582 try:
516   -
  583 +
517 584 # Import matplotlib
518 585 import matplotlib.pyplot as plt
519 586 import matplotlib.gridspec as gridspec
520 587  
521 588 # Create figure
522   - fig = plt.figure(figsize=(15.0,7.8))
  589 + ysize = 7.0
  590 + xsize = 1.4142 * ysize
  591 + fig = plt.figure(figsize=(xsize, ysize))
523 592  
524 593 # Create title and subtitle
525   - title = 'ctools carbon footprint report'
  594 + user = pwd.getpwuid(os.getuid())[0]
  595 + title = 'ctools carbon footprint report for user "%s"' % (user)
526 596 subtitle = r'Dates: %s - %s' % \
527 597 (statistics['use_start'], statistics['use_stop'])
528 598 fig.suptitle(title, fontsize=16)
529   - fig.text(0.5, 0.93, subtitle, fontsize=12, ha='center')
  599 + fig.text(0.5, 0.925, subtitle, fontsize=11, ha='center')
530 600  
531 601 # Set plot margins
532   - fig.subplots_adjust(left=0.07, bottom=0.07, right=0.97, top=0.88, wspace=0.3, hspace=0.35)
  602 + fig.subplots_adjust(left=0.07, bottom=0.07, right=0.97, top=0.88,
  603 + wspace=0.1, hspace=0.2)
533 604  
534 605 # Divide figure
535   - gs1 = gridspec.GridSpec(3,3)
  606 + gs1 = gridspec.GridSpec(4,3) # (rows,cols)
  607 + gs1.update(hspace=0.8, wspace=0.8)
536 608 ax1 = fig.add_subplot(gs1[0,0:2])
537   - ax2 = fig.add_subplot(gs1[0,2])
538   -
539   - # Plot daily footprint
540   - self._plot_daily(ax1, statistics)
541   -
542   - #plt.show()
  609 + ax2 = fig.add_subplot(gs1[1,0:2])
  610 + ax3 = fig.add_subplot(gs1[2,0:2])
  611 + ax4 = fig.add_subplot(gs1[3,0:2])
  612 + #
  613 + gs2 = gridspec.GridSpec(8,3) # (rows,cols)
  614 + gs2.update(hspace=0.3, bottom=0.0, top=0.95, right=0.96, wspace=0.0)
  615 + ax10 = fig.add_subplot(gs2[2:5,2])
  616 + ax11 = fig.add_subplot(gs2[5:8,2])
  617 +
  618 + # Plot daily figures
  619 + self._plot_daily(ax1, statistics, quantity='gCO2e', title='Footprint',
  620 + ylabel=r'g CO$_2$e')
  621 + self._plot_daily(ax2, statistics, quantity='cpu', title='CPU hours',
  622 + ylabel='hours', yscale=1.0/3600.0)
  623 + self._plot_daily(ax3, statistics, quantity='wall', title='Wall clock hours',
  624 + ylabel='hours', yscale=1.0/3600.0)
  625 + self._plot_daily(ax4, statistics, quantity='calls',
  626 + title='ctools calls', ylabel='calls')
  627 +
  628 + # Plot pie figures
  629 + self._plot_pie(ax10, statistics, quantity='gCO2e', title='Footprint')
  630 + self._plot_pie(ax11, statistics, quantity='calls', title='ctools calls')
  631 +
  632 + # Derive summary information from statistics
  633 + info = self._get_global_information(statistics, latex=True)
  634 +
  635 + # Write summary information
  636 + x0 = 0.63
  637 + y0 = 0.89
  638 + dx = 0.17
  639 + dy = 0.022
  640 + fontsize = 8
  641 + fig.text(x0, y0, 'Total carbon footprint: ', fontsize=fontsize, ha='left')
  642 + fig.text(x0+dx, y0, info['gCO2e'], fontsize=fontsize, ha='left')
  643 + y0 -= dy
  644 + fig.text(x0, y0, ' due to power consumption: ', fontsize=fontsize, ha='left')
  645 + fig.text(x0+dx, y0, info['elect'], fontsize=fontsize, ha='left')
  646 + y0 -= dy
  647 + fig.text(x0, y0, ' due to infrastructure: ', fontsize=fontsize, ha='left')
  648 + fig.text(x0+dx, y0, info['gCO2e_infra'], fontsize=fontsize, ha='left')
  649 + y0 -= dy
  650 + fig.text(x0, y0, 'Total CPU time: ', fontsize=fontsize, ha='left')
  651 + fig.text(x0+dx, y0, info['cpu'], fontsize=fontsize, ha='left')
  652 + y0 -= dy
  653 + fig.text(x0, y0, 'Average carbon intensity: ', fontsize=fontsize, ha='left')
  654 + fig.text(x0+dx, y0, info['ci_cpu'], fontsize=fontsize, ha='left')
  655 + y0 -= dy
  656 + fig.text(x0, y0, 'Average daily footprint: ', fontsize=fontsize, ha='left')
  657 + fig.text(x0+dx, y0, info['fp_dur'], fontsize=fontsize, ha='left')
  658 + y0 -= dy
  659 + fig.text(x0, y0, 'Expected annual footprint: ', fontsize=fontsize, ha='left')
  660 + fig.text(x0+dx, y0, info['fp_yr'], fontsize=fontsize, ha='left')
  661 +
  662 + plt.show()
543 663  
544 664 # Save figure
545 665 fig.savefig(outfile, dpi=300)
... ... @@ -556,10 +676,11 @@ class csfootprint(ctools.cscript):
556 676 # Return
557 677 return
558 678  
559   - # Plot daily footprint
560   - def _plot_daily(self, ax, statistics):
  679 + # Plot daily figure
  680 + def _plot_daily(self, ax, statistics, quantity='gCO2e', title='Footprint',
  681 + ylabel=r'g CO$_2$e', yscale=1.0):
561 682 """
562   - Plot daily footprint
  683 + Plot daily figure
563 684  
564 685 Parameters
565 686 ----------
... ... @@ -567,22 +688,85 @@ class csfootprint(ctools.cscript):
567 688 Plotting frame
568 689 statistics : dict
569 690 Statistics dictionary
  691 + quantity : str, optional
  692 + Quantity to plot
  693 + title : str, optional
  694 + Plot title
  695 + ylabel : str, optional
  696 + Y axis label
  697 + yscale : float, optional
  698 + Y axis scale
570 699 """
571 700 # Create bar data
572   - days = [i for i, _ in enumerate(statistics['daily'])]
573   - data = [entry['gCO2e'] for entry in statistics['daily']]
  701 + days = [i for i, _ in enumerate(statistics['daily'])]
  702 + data = [entry[quantity]*yscale for entry in statistics['daily']]
574 703  
575 704 # Plot bar data
576 705 ax.bar(days, data, 1.0, bottom=0.0, color='red')
577 706  
578 707 # Set labels
579   - ax.set_title('Footprint')
  708 + ax.set_title(title)
580 709 ax.set_xlabel('Days since %s' % statistics['use_start'][0:10])
581   - ax.set_ylabel(r'g CO$_2$e')
582   -
  710 + ax.set_ylabel(ylabel)
  711 +
583 712 # Return
584 713 return
585 714  
  715 + # Plot pie figure
  716 + def _plot_pie(self, ax, statistics, quantity='gCO2e', title='Footprint', num=5):
  717 + """
  718 + Plot pie figure
  719 +
  720 + Parameters
  721 + ----------
  722 + ax : pyplot
  723 + Plotting frame
  724 + statistics : dict
  725 + Statistics dictionary
  726 + quantity : str, optional
  727 + Quantity to plot
  728 + title : str, optional
  729 + Plot title
  730 + num : integer, optional
  731 + Number of pies displayed explicitly
  732 + """
  733 + # Get Python version information
  734 + req_version = (2,4)
  735 + cur_version = sys.version_info
  736 +
  737 + # Optionally sort list
  738 + if cur_version > req_version:
  739 + sorted_entries = sorted(statistics['tools'], key=lambda d: d[quantity], reverse=True)
  740 + else:
  741 + sorted_entries = statistics['tools']
  742 +
  743 + # Create pie data
  744 + labels = []
  745 + sizes = []
  746 + others = 0.0
  747 + for i, entry in enumerate(sorted_entries):
  748 + if i < num:
  749 + labels.append(entry['name'])
  750 + sizes.append(entry[quantity])
  751 + else:
  752 + others += entry[quantity]
  753 + labels.append('others')
  754 + sizes.append(others)
  755 +
  756 + # Plot pie chart
  757 + wedges, _, _ = ax.pie(sizes, autopct='%1.0f%%', pctdistance=0.8,
  758 + startangle=90, radius=1.0,
  759 + wedgeprops=dict(width=0.4, edgecolor='w'))
  760 + ax.axis('equal')
  761 + ax.legend(wedges, labels, loc='center', fontsize=8, frameon=False)
  762 +
  763 + # Set labels
  764 + ax.set_title(title)
  765 +
  766 + # Return
  767 + return
  768 +
  769 +
586 770 # Public methods
587 771 def process(self):
588 772 """
... ...