src/gmxlib/thread_mpi/topology.c

   1 /*
   2 This source code file is part of thread_mpi.
   3 Written by Sander Pronk, Erik Lindahl, and possibly others.
   4
   5 Copyright (c) 2009, Sander Pronk, Erik Lindahl.
   6 All rights reserved.
   7
   8 Redistribution and use in source and binary forms, with or without
   9 modification, are permitted provided that the following conditions are met:
  10 1) Redistributions of source code must retain the above copyright
  11    notice, this list of conditions and the following disclaimer.
  12 2) Redistributions in binary form must reproduce the above copyright
  13    notice, this list of conditions and the following disclaimer in the
  14    documentation and/or other materials provided with the distribution.
  15 3) Neither the name of the copyright holders nor the
  16    names of its contributors may be used to endorse or promote products
  17    derived from this software without specific prior written permission.
  18
  19 THIS SOFTWARE IS PROVIDED BY US ''AS IS'' AND ANY
  20 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  22 DISCLAIMED. IN NO EVENT SHALL WE BE LIABLE FOR ANY
  23 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  24 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  25 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  26 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  28 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 If you want to redistribute modifications, please consider that
  31 scientific software is very special. Version control is crucial -
  32 bugs must be traceable. We will be happy to consider code for
  33 inclusion in the official distribution, but derived work should not
  34 be called official thread_mpi. Details are found in the README & COPYING
  35 files.
  36 */
  37
  38 #ifdef HAVE_TMPI_CONFIG_H
  39 #include "tmpi_config.h"
  40 #endif
  41
  42 #ifdef HAVE_CONFIG_H
  43 #include "config.h"
  44 #endif
  45
  46
  47 #ifdef HAVE_UNISTD_H
  48 #include <unistd.h>
  49 #endif
  50
  51 #include <errno.h>
  52 #include <stdlib.h>
  53 #include <stdio.h>
  54 #include <stdarg.h>
  55 #include <string.h>
  56
  57
  58 #include "impl.h"
  59
  60
  61 /* topology functions */
  62 int tMPI_Topo_test(tMPI_Comm comm, int *status)
  63 {
  64 #ifdef TMPI_TRACE
  65     tMPI_Trace_print("tMPI_Topo_test(%p, %p)", comm, status);
  66 #endif
  67
  68     if (!comm)
  69     {
  70         return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
  71     }
  72
  73     if (comm->cart)
  74         *status=TMPI_CART;
  75     /*else if (comm->graph)
  76         status=MPI_GRAPH;*/
  77     else
  78         *status=TMPI_UNDEFINED;
  79
  80     return TMPI_SUCCESS;
  81 }
  82
  83 int tMPI_Cartdim_get(tMPI_Comm comm, int *ndims)
  84 {
  85 #ifdef TMPI_TRACE
  86     tMPI_Trace_print("tMPI_Cartdim_get(%p, %p)", comm, ndims);
  87 #endif
  88     if (!comm)
  89     {
  90         return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
  91     }
  92     if (!comm->cart || comm->cart->ndims==0)
  93     {
  94         return TMPI_SUCCESS;
  95     }
  96     *ndims=comm->cart->ndims;
  97     return TMPI_SUCCESS;
  98 }
  99
 100
 101 int tMPI_Cart_get(tMPI_Comm comm, int maxdims, int *dims, int *periods,
 102                   int *coords)
 103 {
 104     int i;
 105     int myrank=tMPI_Comm_seek_rank(comm, tMPI_Get_current());
 106
 107 #ifdef TMPI_TRACE
 108     tMPI_Trace_print("tMPI_Cart_get(%p, %d, %p, %p, %p)", comm, maxdims,
 109                        dims, periods, coords);
 110 #endif
 111     if (!comm)
 112     {
 113         return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
 114     }
 115     if (!comm->cart || comm->cart->ndims==0)
 116         return TMPI_SUCCESS;
 117
 118     tMPI_Cart_coords(comm, myrank, maxdims, coords);
 119
 120     for(i=0;i<comm->cart->ndims;i++)
 121     {
 122         if (i>=maxdims)
 123         {
 124             return tMPI_Error(comm, TMPI_ERR_DIMS);
 125         }
 126         dims[i]=comm->cart->dims[i];
 127         periods[i]=comm->cart->periods[i];
 128     }
 129
 130     return TMPI_SUCCESS;
 131 }
 132
 133 int tMPI_Cart_rank(tMPI_Comm comm, int *coords, int *rank)
 134 {
 135     int i,mul=1,ret=0;
 136
 137 #ifdef TMPI_TRACE
 138     tMPI_Trace_print("tMPI_Cart_get(%p, %p, %p)", comm, coords, rank);
 139 #endif
 140     if (!comm)
 141     {
 142         return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
 143     }
 144     if (!comm->cart || comm->cart->ndims==0)
 145         return TMPI_SUCCESS;
 146
 147     /* because of row-major ordering, we count the dimensions down */
 148     for(i=comm->cart->ndims-1;i>=0;i--)
 149     {
 150         int rcoord=coords[i];
 151         if (comm->cart->periods[i])
 152         {
 153             /* apply periodic boundary conditions */
 154             rcoord = rcoord % comm->cart->dims[i];
 155             if (rcoord < 0)
 156                 rcoord += comm->cart->dims[i];
 157         }
 158         else
 159         {
 160             if (rcoord < 0 || rcoord >= comm->cart->dims[i])
 161             {
 162                 return tMPI_Error(comm, TMPI_ERR_DIMS);
 163             }
 164         }
 165         ret += mul*rcoord;
 166         mul *= comm->cart->dims[i];
 167     }
 168     *rank=ret;
 169     return TMPI_SUCCESS;
 170 }
 171
 172 int tMPI_Cart_coords(tMPI_Comm comm, int rank, int maxdims, int *coords)
 173 {
 174     int i;
 175     int rank_left=rank;
 176
 177 #ifdef TMPI_TRACE
 178     tMPI_Trace_print("tMPI_Cart_coords(%p, %d, %d, %p)", comm, rank, maxdims,
 179                      coords);
 180 #endif
 181     if (!comm)
 182     {
 183         return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
 184     }
 185     if (!comm->cart || comm->cart->ndims==0)
 186         return TMPI_SUCCESS;
 187     if (maxdims < comm->cart->ndims)
 188     {
 189         return tMPI_Error(comm, TMPI_ERR_DIMS);
 190     }
 191
 192     /* again, row-major ordering */
 193     for(i=comm->cart->ndims-1;i>=0;i--)
 194     {
 195         coords[i]=rank_left%comm->cart->dims[i];
 196         rank_left /= comm->cart->dims[i];
 197     }
 198
 199     return TMPI_SUCCESS;
 200 }
 201
 202
 203
 204 int tMPI_Cart_map(tMPI_Comm comm, int ndims, int *dims, int *periods,
 205                   int *newrank)
 206 {
 207     /* this function doesn't actually do anything beyond returning the current
 208        rank (or TMPI_UNDEFINED if it doesn't fit in the new topology */
 209     int myrank=tMPI_Comm_seek_rank(comm, tMPI_Get_current());
 210     int Ntot=1;
 211     int i;
 212
 213 #ifdef TMPI_TRACE
 214     tMPI_Trace_print("tMPI_Cart_map(%p, %d, %p, %p, %p)", comm, ndims, dims,
 215                      periods, newrank);
 216 #endif
 217     if (!comm)
 218     {
 219         return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
 220     }
 221     if (!periods)
 222     {
 223         return tMPI_Error(comm, TMPI_ERR_DIMS);
 224     }
 225
 226     /* calculate the total number of procs in cartesian comm */
 227     for(i=0;i<ndims;i++)
 228     {
 229         Ntot *= dims[i];
 230     }
 231
 232     if (myrank >= Ntot)
 233     {
 234         *newrank=TMPI_UNDEFINED;
 235     }
 236     else
 237     {
 238         *newrank=myrank;
 239     }
 240
 241     return TMPI_SUCCESS;
 242 }
 243
 244
 245 /* initialize Cartesian topology info in comm. If ndims==0, dims and periods
 246    are not referenced */
 247 static void tMPI_Cart_init(tMPI_Comm *comm_cart, int ndims, int *dims,
 248                            int *periods)
 249 {
 250     int newrank=-1;
 251     int i;
 252
 253     if (*comm_cart)
 254     {
 255         tMPI_Comm_rank(*comm_cart, &newrank);
 256     }
 257
 258     if (newrank==0)
 259     {
 260         (*comm_cart)->cart=(struct cart_topol*)tMPI_Malloc(
 261                                             sizeof(struct cart_topol));
 262         (*comm_cart)->cart->dims=(int*)tMPI_Malloc(ndims*sizeof(int));
 263         (*comm_cart)->cart->periods=(int*)tMPI_Malloc(ndims*sizeof(int));
 264         (*comm_cart)->cart->ndims=ndims;
 265         for(i=0;i<ndims;i++)
 266         {
 267             (*comm_cart)->cart->dims[i]=dims[i];
 268             (*comm_cart)->cart->periods[i]=periods[i];
 269         }
 270     }
 271
 272     /* and we add a barrier to make sure the cart object is seen by
 273        every thread that is part of the new communicator */
 274     if (*comm_cart)
 275     {
 276         tMPI_Spinlock_barrier_wait( &( (*comm_cart)->barrier) );
 277     }
 278 }
 279
 280 void tMPI_Cart_destroy(struct cart_topol *cart)
 281 {
 282     if (cart)
 283     {
 284         free(cart->dims);
 285         free(cart->periods);
 286     }
 287 }
 288
 289 int tMPI_Cart_create(tMPI_Comm comm_old, int ndims, int *dims, int *periods,
 290                      int reorder, tMPI_Comm *comm_cart)
 291 {
 292     int myrank=tMPI_Comm_seek_rank(comm_old, tMPI_Get_current());
 293     int key=myrank;
 294     int color=0;
 295     int Ntot=1;
 296     int i;
 297
 298
 299 #ifdef TMPI_TRACE
 300     tMPI_Trace_print("tMPI_Cart_create(%p, %d, %p, %p, %d, %p)", comm_old,
 301                      ndims, dims, periods, reorder, comm_cart);
 302 #endif
 303     if (!comm_old)
 304     {
 305         return tMPI_Error(comm_old, TMPI_ERR_COMM);
 306     }
 307     /* calculate the total number of procs in cartesian comm */
 308     for(i=0;i<ndims;i++)
 309     {
 310         Ntot *= dims[i];
 311     }
 312     /* refuse to create if there's not enough procs */
 313     if (comm_old->grp.N < Ntot)
 314     {
 315         *comm_cart=TMPI_COMM_NULL;
 316 #if 1
 317         return tMPI_Error(comm_old, TMPI_ERR_CART_CREATE_NPROCS);
 318 #endif
 319     }
 320
 321     if (key >= Ntot)
 322         key=TMPI_UNDEFINED;
 323
 324     if (reorder)
 325     {
 326         tMPI_Cart_map(comm_old, ndims, dims, periods, &key);
 327     }
 328
 329     if (key==TMPI_UNDEFINED)
 330     {
 331         color=TMPI_UNDEFINED;
 332     }
 333
 334     tMPI_Comm_split(comm_old, color, key, comm_cart);
 335
 336     tMPI_Cart_init(comm_cart, ndims, dims, periods);
 337
 338     return TMPI_SUCCESS;
 339 }
 340
 341
 342 int tMPI_Cart_sub(tMPI_Comm comm, int *remain_dims, tMPI_Comm *newcomm)
 343 {
 344     int myrank;
 345     int ndims=0;
 346     int *dims=NULL;
 347     int *periods=NULL;
 348     int *oldcoords=NULL;
 349     int i;
 350     int ndims_notused=1;
 351     int color_notused=0;
 352
 353 #ifdef TMPI_TRACE
 354     tMPI_Trace_print("tMPI_Cart_sub(%p, %p, %p)", comm, remain_dims, newcomm);
 355 #endif
 356     tMPI_Comm_rank(comm, &myrank);
 357     if ( comm->cart )
 358     {
 359         oldcoords=(int*)tMPI_Malloc(sizeof(int)*comm->cart->ndims);
 360         dims=(int*)tMPI_Malloc(sizeof(int)*comm->cart->ndims);
 361         periods=(int*)tMPI_Malloc(sizeof(int)*comm->cart->ndims);
 362
 363         /* get old coordinates */
 364         tMPI_Cart_coords(comm, myrank, comm->cart->ndims, oldcoords);
 365
 366         for(i=0;i<comm->cart->ndims;i++)
 367         {
 368             if (remain_dims[i])
 369             {
 370                 /* for the remaining dimensions, copy dimensionality data */
 371                 dims[ndims]=comm->cart->dims[i];
 372                 periods[ndims]=comm->cart->periods[i];
 373                 ndims++;
 374             }
 375             else
 376             {
 377                 /* base color on not used coordinates. We keep a
 378                    ndims_notused index multiplier.*/
 379                 color_notused += oldcoords[i]*ndims_notused;
 380                 ndims_notused *= comm->cart->dims[i];
 381             }
 382         }
 383     }
 384
 385     /* key=myrank, because we want the order to remain the same */
 386     tMPI_Comm_split(comm, color_notused, myrank, newcomm);
 387     tMPI_Cart_init(newcomm, ndims, dims, periods);
 388
 389     if (oldcoords)
 390         free(oldcoords);
 391     if (dims)
 392         free(dims);
 393     if (periods)
 394         free(periods);
 395
 396     return TMPI_SUCCESS;
 397 }
 398