src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

   1 /*---------------------------------------------------------------------------*\
   2   =========                 |
   3   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   4    \\    /   O peration     |
   5     \\  /    A nd           | Copyright (C) 2004-2010 OpenCFD Ltd.
   6      \\/     M anipulation  |
   7 -------------------------------------------------------------------------------
   8 License
   9     This file is part of OpenFOAM.
  10
  11     OpenFOAM is free software: you can redistribute it and/or modify it
  12     under the terms of the GNU General Public License as published by
  13     the Free Software Foundation, either version 3 of the License, or
  14     (at your option) any later version.
  15
  16     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
  17     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  18     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  19     for more details.
  20
  21     You should have received a copy of the GNU General Public License
  22     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 \*---------------------------------------------------------------------------*/
  25
  26 #include "UPstream.H"
  27 #include "debug.H"
  28 #include "dictionary.H"
  29 #include "IOstreams.H"
  30
  31 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
  32
  33 defineTypeNameAndDebug(Foam::UPstream, 0);
  34
  35 namespace Foam
  36 {
  37     template<>
  38     const char* Foam::NamedEnum
  39     <
  40         Foam::UPstream::commsTypes,
  41         3
  42     >::names[] =
  43     {
  44         "blocking",
  45         "scheduled",
  46         "nonBlocking"
  47     };
  48 }
  49
  50
  51 const Foam::NamedEnum<Foam::UPstream::commsTypes, 3>
  52     Foam::UPstream::commsTypeNames;
  53
  54
  55 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
  56
  57 void Foam::UPstream::setParRun()
  58 {
  59     parRun_ = true;
  60
  61     Pout.prefix() = '[' +  name(myProcNo()) + "] ";
  62     Perr.prefix() = '[' +  name(myProcNo()) + "] ";
  63 }
  64
  65
  66 void Foam::UPstream::calcLinearComm(const label nProcs)
  67 {
  68     linearCommunication_.setSize(nProcs);
  69
  70     // Master
  71     labelList belowIDs(nProcs - 1);
  72     forAll(belowIDs, i)
  73     {
  74         belowIDs[i] = i + 1;
  75     }
  76
  77     linearCommunication_[0] = commsStruct
  78     (
  79         nProcs,
  80         0,
  81         -1,
  82         belowIDs,
  83         labelList(0)
  84     );
  85
  86     // Slaves. Have no below processors, only communicate up to master
  87     for (label procID = 1; procID < nProcs; procID++)
  88     {
  89         linearCommunication_[procID] = commsStruct
  90         (
  91             nProcs,
  92             procID,
  93             0,
  94             labelList(0),
  95             labelList(0)
  96         );
  97     }
  98 }
  99
 100
 101 // Append my children (and my children children etc.) to allReceives.
 102 void Foam::UPstream::collectReceives
 103 (
 104     const label procID,
 105     const List<DynamicList<label> >& receives,
 106     DynamicList<label>& allReceives
 107 )
 108 {
 109     const DynamicList<label>& myChildren = receives[procID];
 110
 111     forAll(myChildren, childI)
 112     {
 113         allReceives.append(myChildren[childI]);
 114         collectReceives(myChildren[childI], receives, allReceives);
 115     }
 116 }
 117
 118
 119 // Tree like schedule. For 8 procs:
 120 // (level 0)
 121 //      0 receives from 1
 122 //      2 receives from 3
 123 //      4 receives from 5
 124 //      6 receives from 7
 125 // (level 1)
 126 //      0 receives from 2
 127 //      4 receives from 6
 128 // (level 2)
 129 //      0 receives from 4
 130 //
 131 // The sends/receives for all levels are collected per processor (one send per
 132 // processor; multiple receives possible) creating a table:
 133 //
 134 // So per processor:
 135 // proc     receives from   sends to
 136 // ----     -------------   --------
 137 //  0       1,2,4           -
 138 //  1       -               0
 139 //  2       3               0
 140 //  3       -               2
 141 //  4       5               0
 142 //  5       -               4
 143 //  6       7               4
 144 //  7       -               6
 145 void Foam::UPstream::calcTreeComm(label nProcs)
 146 {
 147     label nLevels = 1;
 148     while ((1 << nLevels) < nProcs)
 149     {
 150         nLevels++;
 151     }
 152
 153     List<DynamicList<label> > receives(nProcs);
 154     labelList sends(nProcs, -1);
 155
 156     // Info<< "Using " << nLevels << " communication levels" << endl;
 157
 158     label offset = 2;
 159     label childOffset = offset/2;
 160
 161     for (label level = 0; level < nLevels; level++)
 162     {
 163         label receiveID = 0;
 164         while (receiveID < nProcs)
 165         {
 166             // Determine processor that sends and we receive from
 167             label sendID = receiveID + childOffset;
 168
 169             if (sendID < nProcs)
 170             {
 171                 receives[receiveID].append(sendID);
 172                 sends[sendID] = receiveID;
 173             }
 174
 175             receiveID += offset;
 176         }
 177
 178         offset <<= 1;
 179         childOffset <<= 1;
 180     }
 181
 182     // For all processors find the processors it receives data from
 183     // (and the processors they receive data from etc.)
 184     List<DynamicList<label> > allReceives(nProcs);
 185     for (label procID = 0; procID < nProcs; procID++)
 186     {
 187         collectReceives(procID, receives, allReceives[procID]);
 188     }
 189
 190
 191     treeCommunication_.setSize(nProcs);
 192
 193     for (label procID = 0; procID < nProcs; procID++)
 194     {
 195         treeCommunication_[procID] = commsStruct
 196         (
 197             nProcs,
 198             procID,
 199             sends[procID],
 200             receives[procID].shrink(),
 201             allReceives[procID].shrink()
 202         );
 203     }
 204 }
 205
 206
 207 // Callback from UPstream::init() : initialize linear and tree communication
 208 // schedules now that nProcs is known.
 209 void Foam::UPstream::initCommunicationSchedule()
 210 {
 211     calcLinearComm(nProcs());
 212     calcTreeComm(nProcs());
 213 }
 214
 215
 216 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 217
 218 // Initialise my process number to 0 (the master)
 219 int Foam::UPstream::myProcNo_(0);
 220
 221 // By default this is not a parallel run
 222 bool Foam::UPstream::parRun_(false);
 223
 224 // List of process IDs
 225 Foam::List<int> Foam::UPstream::procIDs_(1, 0);
 226
 227 // Standard transfer message type
 228 int Foam::UPstream::msgType_(1);
 229
 230 // Linear communication schedule
 231 Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::linearCommunication_(0);
 232
 233 // Multi level communication schedule
 234 Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::treeCommunication_(0);
 235
 236 // Should compact transfer be used in which floats replace doubles
 237 // reducing the bandwidth requirement at the expense of some loss
 238 // in accuracy
 239 bool Foam::UPstream::floatTransfer
 240 (
 241     debug::optimisationSwitch("floatTransfer", 0)
 242 );
 243
 244 // Number of processors at which the reduce algorithm changes from linear to
 245 // tree
 246 int Foam::UPstream::nProcsSimpleSum
 247 (
 248     debug::optimisationSwitch("nProcsSimpleSum", 16)
 249 );
 250
 251 // Default commsType
 252 Foam::UPstream::commsTypes Foam::UPstream::defaultCommsType
 253 (
 254     commsTypeNames.read(debug::optimisationSwitches().lookup("commsType"))
 255 );
 256
 257
 258 // ************************************************************************* //