src/gromacs/timing/walltime_accounting.cpp

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2013, The GROMACS development team.
   5  * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
   6  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7  * and including many others, as listed in the AUTHORS file in the
   8  * top-level source directory and at http://www.gromacs.org.
   9  *
  10  * GROMACS is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public License
  12  * as published by the Free Software Foundation; either version 2.1
  13  * of the License, or (at your option) any later version.
  14  *
  15  * GROMACS is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with GROMACS; if not, see
  22  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24  *
  25  * If you want to redistribute modifications to GROMACS, please
  26  * consider that scientific software is very special. Version
  27  * control is crucial - bugs must be traceable. We will be happy to
  28  * consider code for inclusion in the official distribution, but
  29  * derived work must not be called official GROMACS. Details are found
  30  * in the README & COPYING files - if they are missing, get the
  31  * official version at http://www.gromacs.org.
  32  *
  33  * To help us fund GROMACS development, we humbly ask that you cite
  34  * the research papers on the package. Check out http://www.gromacs.org.
  35  */
  36 #include "gmxpre.h"
  37
  38 #include "walltime_accounting.h"
  39
  40 #include "config.h"
  41
  42 #include <ctime>
  43
  44 #ifdef HAVE_UNISTD_H
  45 #include <unistd.h>
  46 #endif
  47 #ifdef HAVE_SYS_TIME_H
  48 #include <sys/time.h>
  49 #endif
  50
  51 #include "gromacs/utility/basedefinitions.h"
  52 #include "gromacs/utility/smalloc.h"
  53
  54 /* TODO in future: convert gmx_walltime_accounting to a class,
  55  * resolve who should have responsibility for recording the number of
  56  * steps done, consider whether parts of finish_time, print_perf,
  57  * wallcycle_print belong in this module.
  58  *
  59  * If/when any kind of task parallelism is implemented (even OpenMP
  60  * regions simultaneously assigned to different tasks), consider
  61  * whether this data structure (and/or cycle counters) should be
  62  * maintained on a per-OpenMP-thread basis.
  63  *
  64  * Consider also replacing this with std::chrono. */
  65
  66 /*! \brief Manages caching wall-clock time measurements for
  67  * simulations */
  68 typedef struct gmx_walltime_accounting {
  69     //! Seconds since the epoch recorded at the start of the simulation
  70     double          start_time_stamp;
  71     /*! \brief Seconds since the epoch recorded at the reset of
  72      * counters for the simulation (or the start, if no reset has
  73      * occured). */
  74     double          reset_time_stamp;
  75     /*! \brief Seconds since the epoch recorded at the reset of
  76      * counters for the simulation for this thread (or the start, if
  77      * no reset has occured). */
  78     double          reset_time_stamp_per_thread;
  79     //! Total seconds elapsed over the simulation since counter reset
  80     double          elapsed_time;
  81     //! Total seconds elapsed over the simulation since counter reset running this thread
  82     double          elapsed_time_over_all_threads;
  83     /*! \brief Number of OpenMP threads that will be launched by this
  84      * MPI rank.
  85      *
  86      * This is used to scale elapsed_time_over_all_threads so
  87      * that any combination of real MPI, thread MPI and OpenMP (even
  88      * mdrun -ntomp_pme) processes/threads would (when run at maximum
  89      * efficiency) return values such that the sum of
  90      * elapsed_time_over_all_threads over all threads was constant
  91      * with respect to parallelism implementation. */
  92     int             numOpenMPThreads;
  93     //! Numbers of steps done before reset of counters
  94     int64_t         nsteps_done_at_reset;
  95     //! Set by integrators to report the amount of work they did
  96     int64_t         nsteps_done;
  97     //! Whether the simulation has finished in a way valid for walltime reporting.
  98     bool            isValidFinish;
  99 } t_gmx_walltime_accounting;
 100
 101 /*! \brief Calls system timing routines (e.g. clock_gettime) to get
 102  * the (fractional) number of seconds elapsed since the epoch when
 103  * this thread was executing.
 104  *
 105  * This can be used to measure system load. This can be unreliable if
 106  * threads migrate between sockets. If thread-specific timers are not
 107  * supported by the OS (e.g. if the OS is not POSIX-compliant), this
 108  * function is implemented by gmx_gettime. */
 109 static double gmx_gettime_per_thread();
 110
 111 // TODO In principle, all this should get protected by checks that
 112 // walltime_accounting is not nullptr. In practice, that nullptr condition
 113 // does not happen, and future refactoring will likely enforce it by
 114 // having the gmx_walltime_accounting_t object be owned by the runner
 115 // object. When these become member functions, existence will be
 116 // guaranteed.
 117
 118 gmx_walltime_accounting_t
 119 walltime_accounting_init(int numOpenMPThreads)
 120 {
 121     gmx_walltime_accounting_t walltime_accounting;
 122
 123     snew(walltime_accounting, 1);
 124     walltime_accounting->start_time_stamp            = 0;
 125     walltime_accounting->reset_time_stamp            = 0;
 126     walltime_accounting->reset_time_stamp_per_thread = 0;
 127     walltime_accounting->elapsed_time                = 0;
 128     walltime_accounting->nsteps_done_at_reset        = 0;
 129     walltime_accounting->nsteps_done                 = 0;
 130     walltime_accounting->numOpenMPThreads            = numOpenMPThreads;
 131     walltime_accounting->isValidFinish               = false;
 132
 133     return walltime_accounting;
 134 }
 135
 136 void
 137 walltime_accounting_destroy(gmx_walltime_accounting_t walltime_accounting)
 138 {
 139     sfree(walltime_accounting);
 140 }
 141
 142 void
 143 walltime_accounting_reset_time(gmx_walltime_accounting_t walltime_accounting,
 144                                int64_t                   step)
 145 {
 146     walltime_accounting->reset_time_stamp            = gmx_gettime();
 147     walltime_accounting->reset_time_stamp_per_thread = gmx_gettime_per_thread();
 148     walltime_accounting->elapsed_time                = 0;
 149     walltime_accounting->nsteps_done                 = 0;
 150     walltime_accounting->nsteps_done_at_reset        = step;
 151 }
 152
 153 void
 154 walltime_accounting_start_time(gmx_walltime_accounting_t walltime_accounting)
 155 {
 156     walltime_accounting_reset_time(walltime_accounting, 0);
 157     walltime_accounting->start_time_stamp = walltime_accounting->reset_time_stamp;
 158 }
 159
 160 void
 161 walltime_accounting_end_time(gmx_walltime_accounting_t walltime_accounting)
 162 {
 163     double now, now_per_thread;
 164
 165     now            = gmx_gettime();
 166     now_per_thread = gmx_gettime_per_thread();
 167
 168     walltime_accounting->elapsed_time                  = now - walltime_accounting->reset_time_stamp;
 169     walltime_accounting->elapsed_time_over_all_threads = now_per_thread - walltime_accounting->reset_time_stamp_per_thread;
 170     /* For thread-MPI, the per-thread CPU timer makes this just
 171      * work. For OpenMP threads, the per-thread CPU timer measurement
 172      * needs to be multiplied by the number of OpenMP threads used,
 173      * under the current assumption that all regions ever opened
 174      * within a process are of the same size, and each thread should
 175      * keep one core busy.
 176      */
 177     walltime_accounting->elapsed_time_over_all_threads *= walltime_accounting->numOpenMPThreads;
 178 }
 179
 180 double
 181 walltime_accounting_get_time_since_start(gmx_walltime_accounting_t walltime_accounting)
 182 {
 183     return gmx_gettime() - walltime_accounting->start_time_stamp;
 184 }
 185
 186 double
 187 walltime_accounting_get_time_since_reset(gmx_walltime_accounting_t walltime_accounting)
 188 {
 189     return gmx_gettime() - walltime_accounting->reset_time_stamp;
 190 }
 191
 192 double
 193 walltime_accounting_get_time_since_reset_over_all_threads(gmx_walltime_accounting_t walltime_accounting)
 194 {
 195     return walltime_accounting->elapsed_time_over_all_threads;
 196 }
 197
 198 double
 199 walltime_accounting_get_start_time_stamp(gmx_walltime_accounting_t walltime_accounting)
 200 {
 201     return walltime_accounting->start_time_stamp;
 202 }
 203
 204 int64_t
 205 walltime_accounting_get_nsteps_done_since_reset(gmx_walltime_accounting_t walltime_accounting)
 206 {
 207     return walltime_accounting->nsteps_done - walltime_accounting->nsteps_done_at_reset;
 208 }
 209
 210 void
 211 walltime_accounting_set_nsteps_done(gmx_walltime_accounting_t   walltime_accounting,
 212                                     int64_t                     nsteps_done)
 213 {
 214     walltime_accounting->nsteps_done = nsteps_done;
 215 }
 216
 217 double
 218 gmx_gettime()
 219 {
 220     /* Use clock_gettime only if we know linking the C run-time
 221        library will work (which is not trivial on e.g. Crays), and its
 222        headers claim sufficient support for POSIX (ie not Mac and
 223        Windows). */
 224 #if HAVE_CLOCK_GETTIME && defined(_POSIX_TIMERS) && _POSIX_TIMERS > 0
 225     struct timespec t;
 226     double          seconds;
 227
 228     clock_gettime(CLOCK_REALTIME, &t);
 229     seconds = static_cast<double>(t.tv_sec) + 1e-9*t.tv_nsec;
 230
 231     return seconds;
 232 #elif HAVE_GETTIMEOFDAY
 233     // Note that gettimeofday() is deprecated by POSIX, but since Mac
 234     // and Windows do not yet support POSIX, we are still stuck.
 235     struct timeval t;
 236     double         seconds;
 237
 238     gettimeofday(&t, nullptr);
 239     seconds = static_cast<double>(t.tv_sec) + 1e-6*t.tv_usec;
 240
 241     return seconds;
 242 #else
 243     double  seconds;
 244
 245     seconds = time(nullptr);
 246
 247     return seconds;
 248 #endif
 249 }
 250
 251 void
 252 walltime_accounting_set_valid_finish(gmx_walltime_accounting_t walltime_accounting)
 253 {
 254     walltime_accounting->isValidFinish = true;
 255 }
 256
 257 //! Return whether the simulation finished in a way valid for reporting walltime.
 258 bool
 259 walltime_accounting_get_valid_finish(const gmx_walltime_accounting* walltime_accounting)
 260 {
 261     return walltime_accounting->isValidFinish;
 262 }
 263
 264 static double
 265 gmx_gettime_per_thread()
 266 {
 267     /* Use clock_gettime only if we know linking the C run-time
 268        library will work (which is not trivial on e.g. Crays), and its
 269        headers claim sufficient support for POSIX (ie not Mac and
 270        Windows). */
 271 #if HAVE_CLOCK_GETTIME && defined(_POSIX_THREAD_CPUTIME) && _POSIX_THREAD_CPUTIME > 0
 272     struct timespec t;
 273     double          seconds;
 274
 275     clock_gettime(CLOCK_THREAD_CPUTIME_ID, &t);
 276     seconds = static_cast<double>(t.tv_sec) + 1e-9*t.tv_nsec;
 277
 278     return seconds;
 279 #else
 280     return gmx_gettime();
 281 #endif
 282 }