src/gromacs/mdrunutility/multisim.cpp

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2018,2019, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*! \internal \file
  36  *
  37  * \brief Implements the multi-simulation support routines.
  38  *
  39  * \author Mark Abraham <mark.j.abraham@gmail.com>
  40  * \ingroup module_mdrunutility
  41  */
  42 #include "gmxpre.h"
  43
  44 #include "multisim.h"
  45
  46 #include "config.h"
  47
  48 #include "gromacs/gmxlib/network.h"
  49 #include "gromacs/mdtypes/commrec.h"
  50 #include "gromacs/utility/fatalerror.h"
  51 #include "gromacs/utility/futil.h"
  52 #include "gromacs/utility/mpiinplacebuffers.h"
  53 #include "gromacs/utility/smalloc.h"
  54
  55 gmx_multisim_t *init_multisystem(MPI_Comm                         comm,
  56                                  gmx::ArrayRef<const std::string> multidirs)
  57 {
  58     gmx_multisim_t *ms;
  59 #if GMX_MPI
  60     MPI_Group       mpi_group_world;
  61     int            *rank;
  62 #endif
  63
  64     if (multidirs.empty())
  65     {
  66         return nullptr;
  67     }
  68
  69     if (!GMX_LIB_MPI && !multidirs.empty())
  70     {
  71         gmx_fatal(FARGS, "mdrun -multidir is only supported when GROMACS has been "
  72                   "configured with a proper external MPI library.");
  73     }
  74
  75     if (multidirs.size() == 1)
  76     {
  77         /* NOTE: It would be nice if this special case worked, but this requires checks/tests. */
  78         gmx_fatal(FARGS, "To run mdrun in multiple simulation mode, more then one "
  79                   "actual simulation is required. The single simulation case is not supported.");
  80     }
  81
  82 #if GMX_MPI
  83     int numRanks;
  84     MPI_Comm_size(comm, &numRanks);
  85     if (numRanks % multidirs.size() != 0)
  86     {
  87         gmx_fatal(FARGS, "The number of ranks (%d) is not a multiple of the number of simulations (%td)", numRanks, multidirs.ssize());
  88     }
  89
  90     int numRanksPerSim = numRanks/multidirs.size();
  91     int rankWithinComm;
  92     MPI_Comm_rank(comm, &rankWithinComm);
  93
  94     if (debug)
  95     {
  96         fprintf(debug, "We have %td simulations, %d ranks per simulation, local simulation is %d\n", multidirs.ssize(), numRanksPerSim, rankWithinComm/numRanksPerSim);
  97     }
  98
  99     ms       = new gmx_multisim_t;
 100     ms->nsim = multidirs.size();
 101     ms->sim  = rankWithinComm/numRanksPerSim;
 102     /* Create a communicator for the master nodes */
 103     snew(rank, ms->nsim);
 104     for (int i = 0; i < ms->nsim; i++)
 105     {
 106         rank[i] = i*numRanksPerSim;
 107     }
 108     MPI_Comm_group(comm, &mpi_group_world);
 109     MPI_Group_incl(mpi_group_world, ms->nsim, rank, &ms->mpi_group_masters);
 110     sfree(rank);
 111     MPI_Comm_create(MPI_COMM_WORLD, ms->mpi_group_masters,
 112                     &ms->mpi_comm_masters);
 113
 114 #if !MPI_IN_PLACE_EXISTS
 115     /* initialize the MPI_IN_PLACE replacement buffers */
 116     snew(ms->mpb, 1);
 117     ms->mpb->ibuf        = nullptr;
 118     ms->mpb->libuf       = nullptr;
 119     ms->mpb->fbuf        = nullptr;
 120     ms->mpb->dbuf        = nullptr;
 121     ms->mpb->ibuf_alloc  = 0;
 122     ms->mpb->libuf_alloc = 0;
 123     ms->mpb->fbuf_alloc  = 0;
 124     ms->mpb->dbuf_alloc  = 0;
 125 #endif
 126
 127     // TODO This should throw upon error
 128     gmx_chdir(multidirs[ms->sim].c_str());
 129 #else
 130     GMX_UNUSED_VALUE(comm);
 131     ms = nullptr;
 132 #endif
 133
 134     return ms;
 135 }
 136
 137 void done_multisim(gmx_multisim_t *ms)
 138 {
 139     if (ms == nullptr)
 140     {
 141         return;
 142     }
 143     done_mpi_in_place_buf(ms->mpb);
 144
 145 #if GMX_MPI
 146     // TODO This would work better if the result of MPI_Comm_split was
 147     // put into an RAII-style guard, such as gmx::unique_cptr.
 148     if (ms->mpi_comm_masters != MPI_COMM_NULL &&
 149         ms->mpi_comm_masters != MPI_COMM_WORLD)
 150     {
 151         MPI_Comm_free(&ms->mpi_comm_masters);
 152     }
 153     if (ms->mpi_group_masters != MPI_GROUP_NULL)
 154     {
 155         MPI_Group_free(&ms->mpi_group_masters);
 156     }
 157 #endif
 158     delete ms;
 159 }
 160
 161 #if GMX_MPI
 162 static void gmx_sumd_comm(int nr, double r[], MPI_Comm mpi_comm)
 163 {
 164 #if MPI_IN_PLACE_EXISTS
 165     MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM, mpi_comm);
 166 #else
 167     /* this function is only used in code that is not performance critical,
 168        (during setup, when comm_rec is not the appropriate communication
 169        structure), so this isn't as bad as it looks. */
 170     double *buf;
 171     int     i;
 172
 173     snew(buf, nr);
 174     MPI_Allreduce(r, buf, nr, MPI_DOUBLE, MPI_SUM, mpi_comm);
 175     for (i = 0; i < nr; i++)
 176     {
 177         r[i] = buf[i];
 178     }
 179     sfree(buf);
 180 #endif
 181 }
 182 #endif
 183
 184 #if GMX_MPI
 185 static void gmx_sumf_comm(int nr, float r[], MPI_Comm mpi_comm)
 186 {
 187 #if MPI_IN_PLACE_EXISTS
 188     MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, mpi_comm);
 189 #else
 190     /* this function is only used in code that is not performance critical,
 191        (during setup, when comm_rec is not the appropriate communication
 192        structure), so this isn't as bad as it looks. */
 193     float *buf;
 194     int    i;
 195
 196     snew(buf, nr);
 197     MPI_Allreduce(r, buf, nr, MPI_FLOAT, MPI_SUM, mpi_comm);
 198     for (i = 0; i < nr; i++)
 199     {
 200         r[i] = buf[i];
 201     }
 202     sfree(buf);
 203 #endif
 204 }
 205 #endif
 206
 207 void gmx_sumd_sim(int gmx_unused nr, double gmx_unused r[], const gmx_multisim_t gmx_unused *ms)
 208 {
 209 #if !GMX_MPI
 210     gmx_call("gmx_sumd_sim");
 211 #else
 212     gmx_sumd_comm(nr, r, ms->mpi_comm_masters);
 213 #endif
 214 }
 215
 216 void gmx_sumf_sim(int gmx_unused nr, float gmx_unused r[], const gmx_multisim_t gmx_unused *ms)
 217 {
 218 #if !GMX_MPI
 219     gmx_call("gmx_sumf_sim");
 220 #else
 221     gmx_sumf_comm(nr, r, ms->mpi_comm_masters);
 222 #endif
 223 }
 224
 225 void gmx_sumi_sim(int gmx_unused nr, int gmx_unused r[], const gmx_multisim_t gmx_unused *ms)
 226 {
 227 #if !GMX_MPI
 228     gmx_call("gmx_sumi_sim");
 229 #else
 230 #if MPI_IN_PLACE_EXISTS
 231     MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
 232 #else
 233     /* this is thread-unsafe, but it will do for now: */
 234     int i;
 235
 236     if (nr > ms->mpb->ibuf_alloc)
 237     {
 238         ms->mpb->ibuf_alloc = nr;
 239         srenew(ms->mpb->ibuf, ms->mpb->ibuf_alloc);
 240     }
 241     MPI_Allreduce(r, ms->mpb->ibuf, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
 242     for (i = 0; i < nr; i++)
 243     {
 244         r[i] = ms->mpb->ibuf[i];
 245     }
 246 #endif
 247 #endif
 248 }
 249
 250 void gmx_sumli_sim(int gmx_unused nr, int64_t gmx_unused r[], const gmx_multisim_t gmx_unused *ms)
 251 {
 252 #if !GMX_MPI
 253     gmx_call("gmx_sumli_sim");
 254 #else
 255 #if MPI_IN_PLACE_EXISTS
 256     MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT64_T, MPI_SUM,
 257                   ms->mpi_comm_masters);
 258 #else
 259     /* this is thread-unsafe, but it will do for now: */
 260     int i;
 261
 262     if (nr > ms->mpb->libuf_alloc)
 263     {
 264         ms->mpb->libuf_alloc = nr;
 265         srenew(ms->mpb->libuf, ms->mpb->libuf_alloc);
 266     }
 267     MPI_Allreduce(r, ms->mpb->libuf, nr, MPI_INT64_T, MPI_SUM,
 268                   ms->mpi_comm_masters);
 269     for (i = 0; i < nr; i++)
 270     {
 271         r[i] = ms->mpb->libuf[i];
 272     }
 273 #endif
 274 #endif
 275 }
 276
 277 void check_multi_int(FILE *log, const gmx_multisim_t *ms, int val,
 278                      const char *name,
 279                      gmx_bool bQuiet)
 280 {
 281     int     *ibuf, p;
 282     gmx_bool bCompatible;
 283
 284     if (nullptr != log && !bQuiet)
 285     {
 286         fprintf(log, "Multi-checking %s ... ", name);
 287     }
 288
 289     if (ms == nullptr)
 290     {
 291         gmx_fatal(FARGS,
 292                   "check_multi_int called with a NULL communication pointer");
 293     }
 294
 295     snew(ibuf, ms->nsim);
 296     ibuf[ms->sim] = val;
 297     gmx_sumi_sim(ms->nsim, ibuf, ms);
 298
 299     bCompatible = TRUE;
 300     for (p = 1; p < ms->nsim; p++)
 301     {
 302         bCompatible = bCompatible && (ibuf[p-1] == ibuf[p]);
 303     }
 304
 305     if (bCompatible)
 306     {
 307         if (nullptr != log && !bQuiet)
 308         {
 309             fprintf(log, "OK\n");
 310         }
 311     }
 312     else
 313     {
 314         if (nullptr != log)
 315         {
 316             fprintf(log, "\n%s is not equal for all subsystems\n", name);
 317             for (p = 0; p < ms->nsim; p++)
 318             {
 319                 fprintf(log, "  subsystem %d: %d\n", p, ibuf[p]);
 320             }
 321         }
 322         gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->nsim);
 323     }
 324
 325     sfree(ibuf);
 326 }
 327
 328 void check_multi_int64(FILE *log, const gmx_multisim_t *ms,
 329                        int64_t val, const char *name,
 330                        gmx_bool bQuiet)
 331 {
 332     int64_t          *ibuf;
 333     int               p;
 334     gmx_bool          bCompatible;
 335
 336     if (nullptr != log && !bQuiet)
 337     {
 338         fprintf(log, "Multi-checking %s ... ", name);
 339     }
 340
 341     if (ms == nullptr)
 342     {
 343         gmx_fatal(FARGS,
 344                   "check_multi_int called with a NULL communication pointer");
 345     }
 346
 347     snew(ibuf, ms->nsim);
 348     ibuf[ms->sim] = val;
 349     gmx_sumli_sim(ms->nsim, ibuf, ms);
 350
 351     bCompatible = TRUE;
 352     for (p = 1; p < ms->nsim; p++)
 353     {
 354         bCompatible = bCompatible && (ibuf[p-1] == ibuf[p]);
 355     }
 356
 357     if (bCompatible)
 358     {
 359         if (nullptr != log && !bQuiet)
 360         {
 361             fprintf(log, "OK\n");
 362         }
 363     }
 364     else
 365     {
 366         // TODO Part of this error message would also be good to go to
 367         // stderr (from one rank of one sim only)
 368         if (nullptr != log)
 369         {
 370             fprintf(log, "\n%s is not equal for all subsystems\n", name);
 371             for (p = 0; p < ms->nsim; p++)
 372             {
 373                 char strbuf[255];
 374                 /* first make the format string */
 375                 snprintf(strbuf, 255, "  subsystem %%d: %s\n",
 376                          "%" PRId64);
 377                 fprintf(log, strbuf, p, ibuf[p]);
 378             }
 379         }
 380         gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->nsim);
 381     }
 382
 383     sfree(ibuf);
 384 }
 385
 386 bool isMasterSim(const gmx_multisim_t *ms)
 387 {
 388     return !isMultiSim(ms) || ms->sim == 0;
 389 }
 390
 391 bool isMasterSimMasterRank(const gmx_multisim_t *ms,
 392                            const bool            isMaster)
 393 {
 394     return (isMaster && isMasterSim(ms));
 395 }