diff --git a/cscripts/Makefile.am b/cscripts/Makefile.am index ab76f3d..9e752e1 100755 --- a/cscripts/Makefile.am +++ b/cscripts/Makefile.am @@ -1,7 +1,7 @@ ############################################################################# # Makefile for cscripts scripts # # ------------------------------------------------------------------------- # -# Copyright (C) 2016-2021 Juergen Knoedlseder # +# Copyright (C) 2016-2022 Juergen Knoedlseder # # ------------------------------------------------------------------------- # # # # This program is free software: you can redistribute it and/or modify # @@ -25,6 +25,7 @@ pythonscripts = $(top_builddir)/cscripts/__init__.py \ $(srcdir)/csbkgmodel.py \ $(srcdir)/cscaldb.py \ $(srcdir)/csebins.py \ + $(srcdir)/csfootprint.py \ $(srcdir)/csinfo.py \ $(srcdir)/cslightcrv.py \ $(srcdir)/csmodelinfo.py \ @@ -88,6 +89,7 @@ parfiles = $(srcdir)/csadd2caldb.par \ $(srcdir)/cstsmapsplit.par \ $(srcdir)/csviscube.par \ $(srcdir)/csscs.par \ + $(srcdir)/csfootprint.par \ $(srcdir)/csworkflow.par \ $(srcdir)/csfindobs.par \ $(srcdir)/csiactcopy.par \ @@ -123,6 +125,7 @@ linkscripts = csadd2caldb \ cstsmapsplit \ csviscube \ csscs \ + csfootprint \ csworkflow \ csfindobs \ csiactcopy \ @@ -163,6 +166,7 @@ testscripts = $(srcdir)/tests/__init__.py \ $(top_srcdir)/test/test_cstsmapsplit.py \ $(top_srcdir)/test/test_csviscube.py \ $(top_srcdir)/test/test_csscs.py \ + $(top_srcdir)/test/test_csfootprint.py \ $(top_srcdir)/test/test_csworkflow.py \ $(top_srcdir)/test/test_obsutils.py \ $(top_srcdir)/test/test_mputils.py \ diff --git a/cscripts/__init__.py.in b/cscripts/__init__.py.in index 6d28d1d..9480f1d 100755 --- a/cscripts/__init__.py.in +++ b/cscripts/__init__.py.in @@ -1,7 +1,7 @@ # ========================================================================== # cscripts Python module # -# Copyright (C) 2015-2021 Juergen Knoedlseder +# Copyright (C) 2015-2022 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +28,7 @@ __all__ = [ "csbkgmodel", "cscaldb", "csebins", + "csfootprint", "csinfo", "cslightcrv", "csmodelinfo", @@ -94,6 +95,7 @@ from cscripts.cstsmapmerge import cstsmapmerge from cscripts.cstsmapsplit import cstsmapsplit from cscripts.csviscube import csviscube from cscripts.csscs import csscs +from cscripts.csfootprint import csfootprint from cscripts.csworkflow import csworkflow from cscripts import obsutils from cscripts import modutils diff --git a/cscripts/csfootprint.par b/cscripts/csfootprint.par new file mode 100644 index 0000000..af3b6fc --- /dev/null +++ b/cscripts/csfootprint.par @@ -0,0 +1,35 @@ +############################################################################# +# # +# Science Analysis Tools # +# # +# ------------------------------------------------------------------------- # +# # +# File: csfootprint.par # +# # +# Author: Juergen Knoedlseder # +# Jurgen.Knodlseder@irap.omp.eu # +# IRAP # +# # +# Purpose: Parameter file for the csfootprint script # +# # +############################################################################# +# +# Input and output parameters +#============================ +infile, f, h, $HOME/.gamma/statistics.xml,,, "Input high-level statistics XML file" +outfile, f, a, NONE,,, "Output graphics file (NONE if no graphics should be generated)" + +# +# Script parameters +#================== +tmin, t, a, NONE,,, "Start time for report (UTC string, JD, MJD or MET in seconds)" +tmax, t, a, NONE,,, "Stop time for report (UTC string, JD, MJD or MET in seconds)" + +# +# Standard parameters +#==================== +chatter, i, h, 2,0,4, "Chattiness of output" +clobber, b, h, yes,,, "Overwrite existing output files with new output files?" +debug, b, h, no,,, "Activate debugging mode?" +mode, s, h, ql,,, "Mode of automatic parameters" +logfile, f, h, csfootprint.log,,, "Log filename" diff --git a/cscripts/csfootprint.py b/cscripts/csfootprint.py new file mode 100755 index 0000000..57f549d --- /dev/null +++ b/cscripts/csfootprint.py @@ -0,0 +1,551 @@ +#! /usr/bin/env python +# ========================================================================== +# Carbon footprint report script +# +# Copyright (C) 2022 Juergen Knoedlseder +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ========================================================================== +import sys +import gammalib +import ctools + + +# ================= # +# csfootprint class # +# ================= # +class csfootprint(ctools.cscript): + """ + Carbon footprint report script + """ + + # Constructor + def __init__(self, *argv): + """ + Constructor + + Parameters + ---------- + argv : list of str + List of IRAF command line parameter strings of the form + ``parameter=3``. + """ + # Initialise application by calling the base class constructor + self._init_cscript(self.__class__.__name__, ctools.__version__, argv) + + # Return + return + + # State methods for pickling + def __getstate__(self): + """ + Extend ctools.cscript __getstate__ method + + Returns + ------- + state : dict + Pickled instance + """ + # Set pickled dictionary + state = {'base' : ctools.cscript.__getstate__(self)} + + # Return pickled dictionary + return state + + def __setstate__(self, state): + """ + Extend ctools.cscript __setstate__ method + + Parameters + ---------- + state : dict + Pickled instance + """ + # Set state + ctools.cscript.__setstate__(self, state['base']) + + # Return + return + + + # Private methods + def _get_parameters(self): + """ + Get parameters from parfile + """ + # Query input parameters + self['infile'].filename() + if self['tmin'].is_valid(): + self['tmin'].time() + if self['tmax'].is_valid(): + self['tmax'].time() + + # Query ahead output model filename + if self._read_ahead(): + if self['outfile'].is_valid(): + self['outfile'].filename() + + # Write input parameters into logger + self._log_parameters(gammalib.TERSE) + + # Return + return + + # Load statistics + def _load_statistics(self): + """ + Load statistics + """ + # Initialise statistics + statistics = {} + + # Get filename + infile = self['infile'].filename() + + # Load XML file + xml = gammalib.GXml() + xml.load(infile) + + # Get useful nodes + header = xml.element('statistics > header') + data = xml.element('statistics > data') + dates = header.element('dates') + countries = data.element('countries') + versions = data.element('versions') + daily = data.element('daily') + + # Extract header dates information + statistics['creation'] = dates.element('creation').string() + statistics['modified'] = dates.element('modified').string() + statistics['start'] = dates.element('start').string() + statistics['stop'] = dates.element('stop').string() + + # Set used time interval [start,stop] + if self['tmin'].is_valid(): + start = self['tmin'].time().utc() + if start < statistics['start']: + start = statistics['start'] + else: + start = statistics['start'] + if self['tmax'].is_valid(): + stop = self['tmax'].time().utc() + if stop > statistics['stop']: + stop = statistics['stop'] + else: + stop = statistics['stop'] + + # Set used time interval + statistics['use_start'] = start + statistics['use_stop'] = stop + + # Extract list of countries + statistics['countries'] = [] + num = countries.elements() + for i in range(num): + element = countries.element(i) + country = element.name() + calls = element.attribute('calls') + wall = element.attribute('wall') + cpu = element.attribute('cpu') + gCO2e = element.attribute('gCO2e') + entry = {'country': country, 'calls': calls, 'wall': wall, + 'cpu': cpu, 'gCO2e': gCO2e} + statistics['countries'].append(entry) + + # Extract list of versions + statistics['versions'] = [] + num = versions.elements() + for i in range(num): + element = versions.element(i) + version = element.name() + calls = element.attribute('calls') + wall = element.attribute('wall') + cpu = element.attribute('cpu') + gCO2e = element.attribute('gCO2e') + entry = {'version': version, 'calls': calls, 'wall': wall, + 'cpu': cpu, 'gCO2e': gCO2e} + statistics['versions'].append(entry) + + # Extract daily and tools statistics + statistics['daily'] = [] + statistics['tools'] = [] + num = daily.elements() + total_calls = 0 + total_wall = 0.0 + total_cpu = 0.0 + total_gCO2e = 0.0 + for i in range(num): + + # Get date + element = daily.element(i) + date = element.element('value',0).string() + + # Continue if date is out of range + if date < start[0:10] or date > stop[0:10]: + continue + + # Collect information + tools = element.element('tools',0) + ntools = tools.elements() + sum_calls = 0 + sum_wall = 0.0 + sum_cpu = 0.0 + sum_gCO2e = 0.0 + for k in range(ntools): + + # Extract information + tool = tools.element(k) + name = tool.name() + calls = int(tool.attribute('calls')) + wall = float(tool.attribute('wall')) + cpu = float(tool.attribute('cpu')) + gCO2e = float(tool.attribute('gCO2e')) + + # Update sums + sum_calls += calls + sum_wall += wall + sum_cpu += cpu + sum_gCO2e += gCO2e + + # Update tools data + if name in statistics['tools']: + statistics['tools'][name]['calls'] += calls + statistics['tools'][name]['wall'] += wall + statistics['tools'][name]['cpu'] += cpu + statistics['tools'][name]['gCO2e'] += gCO2e + else: + entry = {'name': name, 'calls': calls, 'wall': wall, + 'cpu': cpu, 'gCO2e': gCO2e} + statistics['tools'].append(entry) + + # Update totals + total_calls += sum_calls + total_wall += sum_wall + total_cpu += sum_cpu + total_gCO2e += sum_gCO2e + + # Set entry + entry = {'date': date, 'calls': sum_calls, 'wall': sum_wall, + 'cpu': sum_cpu, 'gCO2e': sum_gCO2e} + statistics['daily'].append(entry) + + # Update total statistics + statistics['calls'] = total_calls + statistics['wall'] = total_wall + statistics['cpu'] = total_cpu + statistics['gCO2e'] = total_gCO2e + + # Return statistics + return statistics + + # Format time + def _format_time(self, seconds): + """ + Format time + + Parameters + ---------- + seconds : float + Time in seconds + """ + # Format according to precision + if seconds < 60.0: + format = '%.3f seconds' % (seconds) + elif seconds < 3600.0: + format = '%.3f minutes' % (seconds/60.0) + else: + format = '%.3f hours' % (seconds/3600.0) + + # Return format + return format + + # Format carbon footprint + def _format_footprint(self, gCO2e): + """ + Format carbon footprint + + Parameters + ---------- + gCO2e : float + Carbon footprint in gCO2e + """ + # Format according to precision + if gCO2e < 1000.0: + format = '%.3f g CO2e' % (gCO2e) + elif seconds < 1.0e6: + format = '%.3f kg CO2e' % (gCO2e/1000.0) + else: + format = '%.3f t CO2e' % (gCO2e/1.0e6) + + # Return format + return format + + # Log global statistics + def _global_statistics(self, statistics): + """ + Log global statistics + + Parameters + ---------- + statistics : dict + Statistics dictionary + """ + # Log header + self._log_header1(gammalib.TERSE, 'Global statistics') + + # Derive information + tstart = gammalib.GTime(statistics['use_start']) + tstop = gammalib.GTime(statistics['use_stop']) + duration = tstop - tstart + if statistics['cpu'] != 0.0: + ci_cpu = statistics['gCO2e']/(statistics['cpu']/3600.0) + else: + ci_cpu = 0.0 + if duration != 0.0: + fp_dur = statistics['gCO2e']/(duration * gammalib.sec2day) + else: + fp_dur = 0.0 + fp_yr = fp_dur * 365.25 + dur_str = self._format_time(duration) + if statistics['wall'] != 0.0: + load_str = '%.1f %%' % (statistics['cpu']/statistics['wall']*100.0) + else: + load_str = 'undefined' + ci_cpu_str = self._format_footprint(ci_cpu) + ' / CPU hour' + fp_dur_str = self._format_footprint(fp_dur) + ' / day' + fp_yr_str = self._format_footprint(fp_yr) + ' / year' + wall_str = self._format_time(statistics['wall']) + cpu_str = self._format_time(statistics['cpu']) + gCO2e_str = self._format_footprint(statistics['gCO2e']) + date_str = '%s - %s' % (statistics['start'], statistics['stop']) + use_str = '%s - %s' % (statistics['use_start'], statistics['use_stop']) + + # Log report information + self._log_value(gammalib.NORMAL, 'Creation date', statistics['creation']) + self._log_value(gammalib.NORMAL, 'Last statistics update', statistics['modified']) + self._log_value(gammalib.NORMAL, 'Statistics date interval', date_str) + self._log_value(gammalib.NORMAL, 'Used date interval', use_str) + self._log_value(gammalib.NORMAL, 'Duration of used interval', dur_str) + self._log_value(gammalib.NORMAL, 'Total number of ctool runs', statistics['calls']) + self._log_value(gammalib.NORMAL, 'Total wall clock time', wall_str) + self._log_value(gammalib.NORMAL, 'Total CPU time', cpu_str) + self._log_value(gammalib.NORMAL, 'Average CPU load', load_str) + self._log_value(gammalib.NORMAL, 'Total carbon footprint', gCO2e_str) + self._log_value(gammalib.NORMAL, 'Average carbon intensity', ci_cpu_str) + self._log_value(gammalib.NORMAL, 'Average daily footprint', fp_dur_str) + self._log_value(gammalib.NORMAL, 'Expected annual footprint', fp_yr_str) + + # Return + return + + # Log daily statistics + def _daily_statistics(self, statistics): + """ + Log daily statistics + + Parameters + ---------- + statistics : dict + Statistics dictionary + """ + # Log header + self._log_header1(gammalib.TERSE, 'Daily statistics') + + # Log daily carbon footprint statistics + self._log_header3(gammalib.NORMAL, 'Carbon footprint') + + # Loop over daily entries + for entry in statistics['daily']: + self._log_value(gammalib.NORMAL, entry['date'], self._format_footprint(entry['gCO2e'])) + + # Log daily run statistics + self._log_header3(gammalib.NORMAL, 'ctools or cscript calls') + + # Loop over daily entries + for entry in statistics['daily']: + self._log_value(gammalib.NORMAL, entry['date'], entry['calls']) + + # Log daily carbon footprint statistics + self._log_header3(gammalib.NORMAL, 'Used wall clock time') + + # Loop over daily entries + for entry in statistics['daily']: + self._log_value(gammalib.NORMAL, entry['date'], self._format_time(entry['wall'])) + + # Log daily carbon footprint statistics + self._log_header3(gammalib.NORMAL, 'Used CPU time') + + # Loop over daily entries + for entry in statistics['daily']: + self._log_value(gammalib.NORMAL, entry['date'], self._format_time(entry['cpu'])) + + # Return + return + + # Log tools statistics + def _tools_statistics(self, statistics): + """ + Log tools statistics + + Parameters + ---------- + statistics : dict + Statistics dictionary + """ + # Get Python version information + req_version = (2,4) + cur_version = sys.version_info + + # Log header + self._log_header1(gammalib.TERSE, 'ctools and cscripts statistics') + + # Log daily carbon footprint statistics + self._log_header3(gammalib.NORMAL, 'Carbon footprint') + + # Optionally sort list + if cur_version > req_version: + sorted_entries = sorted(statistics['tools'], key=lambda d: d['gCO2e'], reverse=True) + else: + sorted_entries = statistics['tools'] + + # Loop over entries + for i, entry in enumerate(sorted_entries): + if i < 10: + level = gammalib.NORMAL + else: + level = gammalib.EXPLICIT + self._log_value(level, entry['name'], self._format_footprint(entry['gCO2e'])) + if len(sorted_entries) > 9 and self['chatter'].integer() < 3: + self._log_string(gammalib.NORMAL, ' ... (list truncated after 10 entries) ...') + + # Log daily run statistics + self._log_header3(gammalib.NORMAL, 'ctools or cscript calls') + + # Optionally sort list + if cur_version > req_version: + sorted_entries = sorted(statistics['tools'], key=lambda d: d['calls'], reverse=True) + else: + sorted_entries = statistics['tools'] + + # Loop over entries + for i, entry in enumerate(sorted_entries): + if i < 10: + level = gammalib.NORMAL + else: + level = gammalib.EXPLICIT + self._log_value(level, entry['name'], entry['calls']) + if len(sorted_entries) > 9 and self['chatter'].integer() < 3: + self._log_string(gammalib.NORMAL, ' ... (list truncated after 10 entries) ...') + + # Log daily carbon footprint statistics + self._log_header3(gammalib.NORMAL, 'Used wall clock time') + + # Optionally sort list + if cur_version > req_version: + sorted_entries = sorted(statistics['tools'], key=lambda d: d['wall'], reverse=True) + else: + sorted_entries = statistics['tools'] + + # Loop over entries + for i, entry in enumerate(sorted_entries): + if i < 10: + level = gammalib.NORMAL + else: + level = gammalib.EXPLICIT + self._log_value(level, entry['name'], self._format_time(entry['wall'])) + if len(sorted_entries) > 9 and self['chatter'].integer() < 3: + self._log_string(gammalib.NORMAL, ' ... (list truncated after 10 entries) ...') + + # Log daily carbon footprint statistics + self._log_header3(gammalib.NORMAL, 'Used CPU time') + + # Optionally sort list + if cur_version > req_version: + sorted_entries = sorted(statistics['tools'], key=lambda d: d['cpu'], reverse=True) + else: + sorted_entries = statistics['tools'] + + # Loop over entries + for i, entry in enumerate(sorted_entries): + if i < 10: + level = gammalib.NORMAL + else: + level = gammalib.EXPLICIT + self._log_value(level, entry['name'], self._format_time(entry['cpu'])) + if len(sorted_entries) > 9 and self['chatter'].integer() < 3: + self._log_string(gammalib.NORMAL, ' ... (list truncated after 10 entries) ...') + + + # Return + return + + + # Public methods + def run(self): + """ + Run the script + """ + # Switch screen logging on in debug mode + if self._logDebug(): + self._log.cout(True) + + # Get parameters + self._get_parameters() + + # Log header + self._log_header1(gammalib.TERSE, 'Load statistics data') + + # Load statistics + statistics = self._load_statistics() + + # Log statistics + self._global_statistics(statistics) + self._daily_statistics(statistics) + self._tools_statistics(statistics) + + # Return + return + + def save(self): + """ + Save something + """ + # Write header + self._log_header1(gammalib.TERSE, 'Save graphics') + + # Continue only if filename is valid + if self['outfile'].is_valid(): + + # Implement saving of graphics file + pass + + # ... signal that no graphics file was specified + else: + self._log_value(gammalib.NORMAL, 'Graphics file', 'not specified') + + # Return + return + + +# ======================== # +# Main routine entry point # +# ======================== # +if __name__ == '__main__': + + # Create instance of application + app = csfootprint(sys.argv) + + # Execute application + app.execute() diff --git a/doc/source/users/environment.rst b/doc/source/users/environment.rst index 7c1d516..baf8640 100755 --- a/doc/source/users/environment.rst +++ b/doc/source/users/environment.rst @@ -60,4 +60,104 @@ computation took, and :math:`{\rm Carbon\,\,footprint}` is the carbon footprint The carbon footprint of all ctools and cscript runs are stored in an ASCII file that you will find in a folder under your home directory, named ``.gamma/statistics.csv``. +This file is regularly scanned and purged by the GammaLib daemon that creates the +high-level statistics file ``.gamma/statistics.xml`` that is in the same folder. You may +display the content of this file using the ``csfootprint`` script: + +.. code-block:: bash + + $ csfootprint debug=yes + Start time for report (UTC string, JD, MJD or MET in seconds) [NONE] + Stop time for report (UTC string, JD, MJD or MET in seconds) [NONE] + Output graphics file (NONE if no graphics should be generated) [NONE] + ... + 2022-02-20T14:38:29: +===================+ + 2022-02-20T14:38:29: | Global statistics | + 2022-02-20T14:38:29: +===================+ + 2022-02-20T14:38:29: Creation date .............: 2022-02-19T22:09:05 + 2022-02-20T14:38:29: Last statistics update ....: 2022-02-20T14:35:20 + 2022-02-20T14:38:29: Statistics date interval ..: 2022-02-18T22:21:56 - 2022-02-20T14:35:17 + 2022-02-20T14:38:29: Used date interval ........: 2022-02-18T22:21:56 - 2022-02-20T14:35:17 + 2022-02-20T14:38:29: Duration of used interval .: 40.222 hours + 2022-02-20T14:38:29: Total number of ctool runs : 3104 + 2022-02-20T14:38:29: Total wall clock time .....: 34.200 minutes + 2022-02-20T14:38:29: Total CPU time ............: 27.543 minutes + 2022-02-20T14:38:29: Average CPU load ..........: 80.5 % + 2022-02-20T14:38:29: Total carbon footprint ....: 1.623 g CO2e + 2022-02-20T14:38:29: Average carbon intensity ..: 3.535 g CO2e / CPU hour + 2022-02-20T14:38:29: Average daily footprint ...: 0.968 g CO2e / day + 2022-02-20T14:38:29: Expected annual footprint .: 353.632 g CO2e / year + 2022-02-20T14:38:29: + 2022-02-20T14:38:29: +==================+ + 2022-02-20T14:38:29: | Daily statistics | + 2022-02-20T14:38:29: +==================+ + 2022-02-20T14:38:29: === Carbon footprint === + 2022-02-20T14:38:29: 2022-02-18 ................: 1.336 g CO2e + 2022-02-20T14:38:29: 2022-02-19 ................: 0.010 g CO2e + 2022-02-20T14:38:29: 2022-02-20 ................: 0.277 g CO2e + 2022-02-20T14:38:29: === ctools or cscript calls === + 2022-02-20T14:38:29: 2022-02-18 ................: 2895 + 2022-02-20T14:38:29: 2022-02-19 ................: 80 + 2022-02-20T14:38:29: 2022-02-20 ................: 129 + 2022-02-20T14:38:29: === Used wall clock time === + 2022-02-20T14:38:29: 2022-02-18 ................: 25.483 minutes + 2022-02-20T14:38:29: 2022-02-19 ................: 1.033 minutes + 2022-02-20T14:38:29: 2022-02-20 ................: 7.683 minutes + 2022-02-20T14:38:29: === Used CPU time === + 2022-02-20T14:38:29: 2022-02-18 ................: 22.682 minutes + 2022-02-20T14:38:29: 2022-02-19 ................: 9.895 seconds + 2022-02-20T14:38:29: 2022-02-20 ................: 4.695 minutes + 2022-02-20T14:38:29: + 2022-02-20T14:38:29: +================================+ + 2022-02-20T14:38:29: | ctools and cscripts statistics | + 2022-02-20T14:38:29: +================================+ + 2022-02-20T14:38:29: === Carbon footprint === + 2022-02-20T14:38:29: ctlike ....................: 0.318 g CO2e + 2022-02-20T14:38:29: cspull ....................: 0.186 g CO2e + 2022-02-20T14:38:29: csbkgmodel ................: 0.178 g CO2e + 2022-02-20T14:38:29: ctobssim ..................: 0.172 g CO2e + 2022-02-20T14:38:29: ctobssim ..................: 0.087 g CO2e + 2022-02-20T14:38:29: comobsbin .................: 0.081 g CO2e + 2022-02-20T14:38:29: comlixmap .................: 0.069 g CO2e + 2022-02-20T14:38:29: ctulimit ..................: 0.064 g CO2e + 2022-02-20T14:38:29: cstsdist ..................: 0.058 g CO2e + 2022-02-20T14:38:29: ctselect ..................: 0.044 g CO2e + 2022-02-20T14:38:29: ... (list truncated after 10 entries) ... + 2022-02-20T14:38:29: === ctools or cscript calls === + 2022-02-20T14:38:29: ctobssim ..................: 1795 + 2022-02-20T14:38:29: ctlike ....................: 220 + 2022-02-20T14:38:29: ctselect ..................: 87 + 2022-02-20T14:38:29: csfootprint ...............: 80 + 2022-02-20T14:38:29: cscaldb ...................: 79 + 2022-02-20T14:38:29: csscs .....................: 55 + 2022-02-20T14:38:29: ctulimit ..................: 54 + 2022-02-20T14:38:29: ctcubemask ................: 51 + 2022-02-20T14:38:29: csphagen ..................: 46 + 2022-02-20T14:38:29: ctbin .....................: 31 + 2022-02-20T14:38:29: ... (list truncated after 10 entries) ... + 2022-02-20T14:38:29: === Used wall clock time === + 2022-02-20T14:38:29: ctlike ....................: 5.367 minutes + 2022-02-20T14:38:29: cspull ....................: 3.817 minutes + 2022-02-20T14:38:29: csbkgmodel ................: 3.067 minutes + 2022-02-20T14:38:29: ctobssim ..................: 2.967 minutes + 2022-02-20T14:38:29: csfootprint ...............: 2.300 minutes + 2022-02-20T14:38:29: ctobssim ..................: 1.483 minutes + 2022-02-20T14:38:29: comobsbin .................: 1.367 minutes + 2022-02-20T14:38:29: cstsdist ..................: 1.300 minutes + 2022-02-20T14:38:29: comlixmap .................: 1.183 minutes + 2022-02-20T14:38:29: ctselect ..................: 1.083 minutes + 2022-02-20T14:38:29: ... (list truncated after 10 entries) ... + 2022-02-20T14:38:29: === Used CPU time === + 2022-02-20T14:38:29: ctlike ....................: 5.392 minutes + 2022-02-20T14:38:29: cspull ....................: 3.157 minutes + 2022-02-20T14:38:29: csbkgmodel ................: 3.029 minutes + 2022-02-20T14:38:29: ctobssim ..................: 2.921 minutes + 2022-02-20T14:38:29: ctobssim ..................: 1.469 minutes + 2022-02-20T14:38:29: comobsbin .................: 1.367 minutes + 2022-02-20T14:38:29: comlixmap .................: 1.169 minutes + 2022-02-20T14:38:29: ctulimit ..................: 1.086 minutes + 2022-02-20T14:38:29: cstsdist ..................: 58.978 seconds + 2022-02-20T14:38:29: ctselect ..................: 45.005 seconds + 2022-02-20T14:38:29: ... (list truncated after 10 entries) ... + diff --git a/doc/source/users/reference_manual/csfootprint.rst b/doc/source/users/reference_manual/csfootprint.rst new file mode 100644 index 0000000..5d66f46 --- /dev/null +++ b/doc/source/users/reference_manual/csfootprint.rst @@ -0,0 +1,76 @@ +.. _csfootprint: + +csfootprint +=========== + +Generates a carbon footprint report for the usage of ctools on your computed. + + +Synopsis +-------- + +This script generates a carbon footprint report based on the usage of ctools +on your computed. ctools usage statistics are collected in the file +``~/.gamma/statistics.xml`` that is located in the user's home directory. The +information file is updated using a dedicated daemon that works in the background +and collects information about ctools usage. + + +General parameters +------------------ + +``(infile = $HOME/.gamma/statistics.xml) [file]`` + Input high-level statistics XML file. + +``outfile [file]`` + Output graphics file (``NONE`` if no graphics should be generated). + +``tmin [time]`` + Start time for footprint reporting (UTC string, JD, MJD or MET in seconds). + Start times given in MET seconds are counted with respect to the time + reference of the input observation(s). + If ``INDEF``, ``NONE``, ``UNDEF`` or ``UNDEFINED`` is passed as value, the + report will be generated for the full data provided in the statistics + file. + +``tmax [time]`` + Stop time for footprint reporting (UTC string, JD, MJD or MET in seconds). + Stop times given in MET seconds are counted with respect to the time + reference of the input observation(s). + If ``INDEF``, ``NONE``, ``UNDEF`` or ``UNDEFINED`` is passed as value, the + report will be generated for the full data provided in the statistics + file. + + +Standard parameters +------------------- + +``(chatter = 2) [integer]`` + Verbosity of the executable: + ``chatter = 0``: no information will be logged + + ``chatter = 1``: only errors will be logged + + ``chatter = 2``: errors and actions will be logged + + ``chatter = 3``: report about the task execution + + ``chatter = 4``: detailed report about the task execution + +``(clobber = yes) [boolean]`` + Specifies whether an existing output file should be overwritten. + +``(debug = no) [boolean]`` + Enables debug mode. In debug mode the executable will dump any log file output to the console. + +``(mode = ql) [string]`` + Mode of automatic parameters (default is ``ql``, i.e. "query and learn"). + +``(logfile = csfootprint.log) [string]`` + Name of log file. + + +Related tools or scripts +------------------------ + +None diff --git a/doc/source/users/reference_manual/index.rst b/doc/source/users/reference_manual/index.rst index c96967b..a272fda 100755 --- a/doc/source/users/reference_manual/index.rst +++ b/doc/source/users/reference_manual/index.rst @@ -57,6 +57,7 @@ CTA and IACT analysis tools and scripts ctskymap --- Generates sky map csbkgmodel --- Generates background model for 3D analysis csebins --- Generates energy boundaries for stacked analysis + csfootprint --- Generates carbon footprint report csobsdef --- Generates observation definition file csobsselect --- Select observations from observation definition file csphagen --- Generates PHA, ARF, RMF files based on source/background regions diff --git a/test/data/statistics.xml b/test/data/statistics.xml new file mode 100644 index 0000000..14bc974 --- /dev/null +++ b/test/data/statistics.xml @@ -0,0 +1,113 @@ + + +
+ + 2022-02-19T22:09:05 + 2022-02-20T14:35:20 + 2022-02-18T22:21:56 + 2022-02-20T14:35:17 + + + FR + +
+ + + + 2022-02-18 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2022-02-19 + + + + + + + 2022-02-20 + + + + + + + + + + + + + + + <2.0.0.dev calls="82" wall="138" cpu="1.109517" gCO2e="0.00108955228" /> + + + diff --git a/test/test_csfootprint.py b/test/test_csfootprint.py new file mode 100644 index 0000000..ba59576 --- /dev/null +++ b/test/test_csfootprint.py @@ -0,0 +1,168 @@ +#! /usr/bin/env python +# ========================================================================== +# This scripts performs unit tests for the csfootprint script. +# +# Copyright (C) 2022 Juergen Knoedlseder +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ========================================================================== +import os +import gammalib +import cscripts +from testing import test + + +# ================================= # +# Test class for csfootprint script # +# ================================= # +class Test(test): + """ + Test class for csfootprint script + + This test class makes unit tests for the csfootprint script by using it from + the command line and from Python. + """ + + # Constructor + def __init__(self): + """ + Constructor + """ + # Call base class constructor + test.__init__(self) + + # Set members + self._infile = self._datadir + '/statistics.xml' + + # Return + return + + # Set test functions + def set(self): + """ + Set all test functions + """ + # Set test name + self.name('csfootprint') + + # Append tests + self.append(self._test_cmd, 'Test csfootprint on command line') + self.append(self._test_python, 'Test csfootprint from Python') + self.append(self._test_pickeling, 'Test csfootprint pickeling') + + # Return + return + + # Test csfootprint on command line + def _test_cmd(self): + """ + Test csfootprint on the command line + """ + # Set script name + csfootprint = self._script('csfootprint') + + # Setup csfootprint command + cmd = csfootprint+' infile="'+self._infile+'" outfile=NONE tmin=NONE'+ \ + ' tmax=NONE logfile="csfootprint_cmd1.log" chatter=1' + + # Check if execution was successful + self.test_value(self._execute(cmd), 0, + 'Check successful execution from command line') + + # Setup csfootprint command + cmd = csfootprint+' infile="file_that_does_not_exist.xml" outfile=NONE tmin=NONE'+ \ + ' tmax=NONE logfile="csfootprint_cmd1.log" chatter=1' + + # Check if execution was successful + self.test_assert(self._execute(cmd, success=False) != 0, + 'Check invalid input file when executed from command line') + + # Check csfootprint --help + self._check_help(csfootprint) + + # Return + return + + # Test csfootprint from Python + def _test_python(self): + """ + Test csfootprint from Python + """ + # Allocate csfootprint + script = cscripts.csfootprint() + + # Check that saving does not nothing + script['infile'] = self._infile + script['outfile'] = 'NONE' + script['tmin'] = 'NONE' + script['tmax'] = 'NONE' + script['logfile'] = 'csfootprint_py0.log' + script['chatter'] = 2 + + # Run csfootprint script + script.logFileOpen() + script.run() + + # Check result + self._check_result(script) + + # Return + return + + # Test csfootprint pickeling + def _test_pickeling(self): + """ + Test csfootprint pickeling + """ + # Perform pickeling test of empty class + self._pickeling(cscripts.csfootprint()) + + # Setup script for pickling text + script = cscripts.csfootprint() + script['infile'] = self._infile + script['outfile'] = 'NONE' + script['tmin'] = 'NONE' + script['tmax'] = 'NONE' + script['logfile'] = 'csfootprint_py1_pickle.log' + script['chatter'] = 2 + + # Perform pickeling tests of filled class + obj = self._pickeling(script) + + # Run csfootprint script and save result + obj.logFileOpen() # Make sure we get a log file + obj.run() + obj.save() + + # Check result + self._check_result(script) + + # Return + return + + # Check csfootprint result + def _check_result(self, script): + """ + Check content of script + + Parameters + ---------- + script : `~cscripts.csfootprint` + csfootprint instance + """ + # TODO: Implement test on script result + + # Return + return diff --git a/test/test_python_cscripts.py b/test/test_python_cscripts.py index 1cb4aac..b6f811e 100755 --- a/test/test_python_cscripts.py +++ b/test/test_python_cscripts.py @@ -2,7 +2,7 @@ # ========================================================================== # This scripts performs unit tests for cscripts # -# Copyright (C) 2016-2021 Juergen Knoedlseder +# Copyright (C) 2016-2022 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +28,7 @@ import test_cscaldb import test_csadd2caldb import test_csbkgmodel import test_csebins +import test_csfootprint import test_csfindobs import test_csinfo import test_cslightcrv @@ -146,6 +147,7 @@ def test(installed=False, debug=False): test_cstsmapsplit.Test(), test_csviscube.Test(), test_csscs.Test(), + test_csfootprint.Test(), test_csworkflow.Test(), test_obsutils.Test(), test_mputils.Test()]