1 /*---------------------------------------------------------------------------*\
3 \\ / F ield | foam-extend: Open Source CFD
4 \\ / O peration | Version: 3.2
5 \\ / A nd | Web: http://www.foam-extend.org
6 \\/ M anipulation | For copyright notice see file Copyright
7 -------------------------------------------------------------------------------
9 This file is part of foam-extend.
11 foam-extend is free software: you can redistribute it and/or modify it
12 under the terms of the GNU General Public License as published by the
13 Free Software Foundation, either version 3 of the License, or (at your
14 option) any later version.
16 foam-extend is distributed in the hope that it will be useful, but
17 WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 General Public License for more details.
21 You should have received a copy of the GNU General Public License
22 along with foam-extend. If not, see <http://www.gnu.org/licenses/>.
24 \*---------------------------------------------------------------------------*/
26 #include "ggiGAMGInterface.H"
27 #include "addToRunTimeSelectionTable.H"
29 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
33 defineTypeNameAndDebug(ggiGAMGInterface, 0);
34 addToRunTimeSelectionTable
43 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
45 void Foam::ggiGAMGInterface::initFastReduce() const
47 // Init should be executed only once
50 // Establish parallel comms pattern
51 if (!localParallel() && Pstream::parRun())
53 // Only master handles communication
54 if (Pstream::master())
56 receiveAddr_.setSize(Pstream::nProcs());
57 sendAddr_.setSize(Pstream::nProcs());
59 receiveAddr_[0] = zoneAddressing();
61 for (label procI = 1; procI < Pstream::nProcs(); procI++)
63 // Note: must use normal comms because the size of the
64 // communicated lists is unknown on the receiving side
67 // Opt: reconsider mode of communication
68 IPstream ip(Pstream::scheduled, procI);
70 receiveAddr_[procI] = labelList(ip);
72 sendAddr_[procI] = labelList(ip);
77 // Opt: reconsider mode of communication
78 OPstream op(Pstream::scheduled, Pstream::masterNo());
80 op << zoneAddressing() << shadowInterface().zoneAddressing();
86 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
88 Foam::ggiGAMGInterface::ggiGAMGInterface
90 const lduPrimitiveMesh& lduMesh,
91 const lduInterface& fineInterface,
92 const labelField& localRestrictAddressing,
93 const labelField& neighbourRestrictAddressing
96 GAMGInterface(lduMesh),
97 fineGgiInterface_(refCast<const ggiLduInterface>(fineInterface)),
105 // All processors will do the same coarsening and then filter
106 // the addressing to the local processor
110 // Signalling in global clustering requires me to recognise clustering
111 // from separate processors as separate. In the first phase, this will be
112 // used to recognise cluster from each processor as separate and in the
113 // second phase it will be used to filter local processor faces from
115 // Currently, I am calculating unique cluster index as:
117 // id = cluster + procOffset*myProcID
118 // With procOffset = 1 million, this should be sufficient for 2000 CPUs
119 // with 2 million coarse cells each. For larger numbers, I need a
120 // larger max int, which can be changed on request
123 // Expand the local and neighbour addressing to full zone size
124 labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0);
126 // Memory management, local
128 const labelList& addr = fineGgiInterface_.zoneAddressing();
132 localExpandAddressing[addr[i]] =
133 localRestrictAddressing[i] + procOffset*Pstream::myProcNo();
136 if (!localParallel())
138 reduce(localExpandAddressing, sumOp<labelField>());
142 labelField neighbourExpandAddressing
144 fineGgiInterface_.shadowInterface().zoneSize(),
148 // Memory management, neighbour
150 const labelList& addr =
151 fineGgiInterface_.shadowInterface().zoneAddressing();
155 neighbourExpandAddressing[addr[i]] =
156 neighbourRestrictAddressing[i]
157 + procOffset*Pstream::myProcNo();
160 if (!localParallel())
162 reduce(neighbourExpandAddressing, sumOp<labelField>());
166 // Make a lookup table of entries for owner/neighbour.
167 // All sizes are guessed at the size of fine interface
170 HashTable<SLList<label>, label, Hash<label> > neighboursTable
172 localExpandAddressing.size()
175 // Table of face-sets to be agglomerated
176 HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
178 localExpandAddressing.size()
181 // Table of face-sets weights to be agglomerated
182 HashTable<SLList<SLList<scalar> >, label, Hash<label> >
185 localExpandAddressing.size()
188 // Count the number of coarse faces
189 label nCoarseFaces = 0;
191 // Count the number of agglomeration pairs
192 label nAgglomPairs = 0;
194 // On the fine level, addressing is made in a labelListList
195 if (fineGgiInterface_.fineLevel())
197 const labelListList& fineAddr = fineGgiInterface_.addressing();
198 const scalarListList& fineWeights = fineGgiInterface_.weights();
200 forAll (fineAddr, ffI)
202 const labelList& curFineNbrs = fineAddr[ffI];
203 const scalarList& curFineWeigts = fineWeights[ffI];
205 forAll (curFineNbrs, nbrI)
207 label curMaster = -1;
212 const label nnI = curFineNbrs[nbrI];
213 const scalar curNW = curFineWeigts[nbrI];
215 if (fineGgiInterface_.master())
218 curMaster = localExpandAddressing[ffI];
219 curSlave = neighbourExpandAddressing[nnI];
224 curMaster = neighbourExpandAddressing[nnI];
225 curSlave = localExpandAddressing[ffI];
228 // Look for the master cell. If it has already got a face,
229 // add the coefficient to the face. If not, create a new
231 if (neighboursTable.found(curMaster))
233 // Check all current neighbours to see if the current
234 // slave already exists. If so, add the coefficient.
236 SLList<label>& curNbrs = neighboursTable.find(curMaster)();
238 SLList<SLList<label> >& curFaceFaces =
239 faceFaceTable.find(curMaster)();
241 SLList<SLList<scalar> >& curFaceWeights =
242 faceFaceWeightsTable.find(curMaster)();
244 bool nbrFound = false;
246 SLList<label>::iterator nbrsIter = curNbrs.begin();
248 SLList<SLList<label> >::iterator faceFacesIter =
249 curFaceFaces.begin();
251 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
252 curFaceWeights.begin();
257 nbrsIter != curNbrs.end()
258 && faceFacesIter != curFaceFaces.end()
259 && faceFaceWeightsIter != curFaceWeights.end();
260 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
263 // Check neighbour slave
264 if (nbrsIter() == curSlave)
267 faceFacesIter().append(ffI);
268 faceFaceWeightsIter().append(curNW);
277 curNbrs.append(curSlave);
278 curFaceFaces.append(SLList<label>(ffI));
279 curFaceWeights.append(SLList<scalar>(curNW));
281 // New coarse face created
288 // This master has got no neighbours yet. Add a neighbour
289 // and a coefficient, thus creating a new face
290 neighboursTable.insert(curMaster, SLList<label>(curSlave));
295 SLList<SLList<label> >(SLList<label>(ffI))
298 faceFaceWeightsTable.insert
301 SLList<SLList<scalar> >(SLList<scalar>(curNW))
304 // New coarse face created
308 } // end for all current neighbours
309 } // end for all fine faces
313 // Coarse level, addressing is stored in faceCells
314 forAll (localExpandAddressing, ffi)
316 label curMaster = -1;
319 // Do switching on master/slave indices based on the
320 // owner/neighbour of the processor index such that
321 // both sides get the same answer.
325 curMaster = localExpandAddressing[ffi];
326 curSlave = neighbourExpandAddressing[ffi];
331 curMaster = neighbourExpandAddressing[ffi];
332 curSlave = localExpandAddressing[ffi];
335 // Look for the master cell. If it has already got a face,
336 // add the coefficient to the face. If not, create a new face.
337 if (neighboursTable.found(curMaster))
339 // Check all current neighbours to see if the current slave
340 // already exists and if so, add the fine face
341 // to the agglomeration.
343 SLList<label>& curNbrs = neighboursTable.find(curMaster)();
345 SLList<SLList<label> >& curFaceFaces =
346 faceFaceTable.find(curMaster)();
348 SLList<SLList<scalar> >& curFaceWeights =
349 faceFaceWeightsTable.find(curMaster)();
351 bool nbrFound = false;
353 SLList<label>::iterator nbrsIter = curNbrs.begin();
355 SLList<SLList<label> >::iterator faceFacesIter =
356 curFaceFaces.begin();
358 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
359 curFaceWeights.begin();
364 nbrsIter != curNbrs.end()
365 && faceFacesIter != curFaceFaces.end()
366 && faceFaceWeightsIter != curFaceWeights.end();
367 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
370 // Check neighbour slave
371 if (nbrsIter() == curSlave)
374 faceFacesIter().append(ffi);
376 faceFaceWeightsIter().append(1.0);
384 curNbrs.append(curSlave);
385 curFaceFaces.append(SLList<label>(ffi));
387 curFaceWeights.append(SLList<scalar>(1.0));
389 // New coarse face created
396 // This master has got no neighbours yet. Add a neighbour
397 // and a coefficient, thus creating a new face
398 neighboursTable.insert(curMaster, SLList<label>(curSlave));
403 SLList<SLList<label> >(SLList<label>(ffi))
407 faceFaceWeightsTable.insert
410 SLList<SLList<scalar> >(SLList<scalar>(1.0))
413 // New coarse face created
417 } // end for all fine faces
420 faceCells_.setSize(nCoarseFaces, -1);
421 fineAddressing_.setSize(nAgglomPairs, -1);
422 restrictAddressing_.setSize(nAgglomPairs, -1);
423 restrictWeights_.setSize(nAgglomPairs);
425 labelList contents = neighboursTable.toc();
427 // Sort makes sure the order is identical on both sides.
431 // Grab zone size and create zone addressing
432 zoneSize_ = nCoarseFaces;
434 zoneAddressing_.setSize(nCoarseFaces);
435 label nProcFaces = 0;
437 // Reset face counter for re-use
443 // On master side, the owner addressing is stored in table of contents
444 forAll (contents, masterI)
446 SLList<label>& curNbrs =
447 neighboursTable.find(contents[masterI])();
449 // Note: neighbour processor index is irrelevant. HJ, 1/Apr/2009
451 SLList<SLList<label> >& curFaceFaces =
452 faceFaceTable.find(contents[masterI])();
454 SLList<SLList<scalar> >& curFaceWeights =
455 faceFaceWeightsTable.find(contents[masterI])();
457 SLList<label>::iterator nbrsIter = curNbrs.begin();
458 SLList<SLList<label> >::iterator faceFacesIter =
459 curFaceFaces.begin();
461 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
462 curFaceWeights.begin();
467 nbrsIter != curNbrs.end()
468 && faceFacesIter != curFaceFaces.end()
469 && faceFaceWeightsIter != curFaceWeights.end();
470 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
473 // Check if master is on local processor
476 contents[masterI] >= procOffset*Pstream::myProcNo()
477 && contents[masterI] < procOffset*(Pstream::myProcNo() + 1)
480 // Record that this face belongs locally
481 zoneAddressing_[nProcFaces] = nCoarseFaces;
482 faceCells_[nProcFaces] =
483 contents[masterI] - procOffset*Pstream::myProcNo();
486 SLList<label>::iterator facesIter =
487 faceFacesIter().begin();
488 SLList<scalar>::iterator weightsIter =
489 faceFaceWeightsIter().begin();
494 facesIter != faceFacesIter().end()
495 && weightsIter != faceFaceWeightsIter().end();
496 ++facesIter, ++weightsIter
499 fineAddressing_[nAgglomPairs] = facesIter();
500 restrictAddressing_[nAgglomPairs] = nCoarseFaces;
501 restrictWeights_[nAgglomPairs] = weightsIter();
506 // Not a local face, but still created in global zone
511 // Resize arrays: not all of ggi is used locally
512 faceCells_.setSize(nProcFaces);
513 zoneAddressing_.setSize(nProcFaces);
515 fineAddressing_.setSize(nAgglomPairs);
516 restrictAddressing_.setSize(nAgglomPairs);
517 restrictWeights_.setSize(nAgglomPairs);
521 // On slave side, the owner addressing is stored in linked lists
522 forAll (contents, masterI)
524 // Note: master processor index is irrelevant. HJ, 1/Apr/2009
526 SLList<label>& curNbrs = neighboursTable.find(contents[masterI])();
528 SLList<SLList<label> >& curFaceFaces =
529 faceFaceTable.find(contents[masterI])();
531 SLList<SLList<scalar> >& curFaceWeights =
532 faceFaceWeightsTable.find(contents[masterI])();
534 SLList<label>::iterator nbrsIter = curNbrs.begin();
536 SLList<SLList<label> >::iterator faceFacesIter =
537 curFaceFaces.begin();
539 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
540 curFaceWeights.begin();
544 nbrsIter != curNbrs.end()
545 && faceFacesIter != curFaceFaces.end()
546 && faceFaceWeightsIter != curFaceWeights.end();
547 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
550 // Check if the face is on local processor
553 nbrsIter() >= procOffset*Pstream::myProcNo()
554 && nbrsIter() < procOffset*(Pstream::myProcNo() + 1)
557 // Record that this face belongs locally.
558 zoneAddressing_[nProcFaces] = nCoarseFaces;
559 faceCells_[nProcFaces] =
560 nbrsIter() - procOffset*Pstream::myProcNo();
563 SLList<label>::iterator facesIter =
564 faceFacesIter().begin();
566 SLList<scalar>::iterator weightsIter =
567 faceFaceWeightsIter().begin();
572 facesIter != faceFacesIter().end()
573 && weightsIter != faceFaceWeightsIter().end();
574 ++facesIter, ++weightsIter
577 fineAddressing_[nAgglomPairs] = facesIter();
578 restrictAddressing_[nAgglomPairs] = nCoarseFaces;
579 restrictWeights_[nAgglomPairs] = weightsIter();
588 // Resize arrays: not all of ggi is used locally
589 faceCells_.setSize(nProcFaces);
590 zoneAddressing_.setSize(nProcFaces);
591 fineAddressing_.setSize(nAgglomPairs);
592 restrictAddressing_.setSize(nAgglomPairs);
593 restrictWeights_.setSize(nAgglomPairs);
598 // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
600 Foam::ggiGAMGInterface::~ggiGAMGInterface()
604 // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
606 Foam::tmp<Foam::scalarField> Foam::ggiGAMGInterface::agglomerateCoeffs
608 const scalarField& fineCoeffs
611 // Reassemble fine coefficients to full fine zone size
612 scalarField zoneFineCoeffs(fineGgiInterface_.zoneSize(), 0);
614 const labelList& fineZa = fineGgiInterface_.zoneAddressing();
618 zoneFineCoeffs[fineZa[i]] = fineCoeffs[i];
622 if (!localParallel())
624 reduce(zoneFineCoeffs, sumOp<scalarField>());
627 scalarField zoneCoarseCoeffs(zoneSize(), 0);
629 // Restrict coefficient
630 forAll(restrictAddressing_, ffi)
632 zoneCoarseCoeffs[restrictAddressing_[ffi]] +=
633 restrictWeights_[ffi]*zoneFineCoeffs[fineAddressing_[ffi]];
636 tmp<scalarField> tcoarseCoeffs(new scalarField(size(), 0.0));
637 scalarField& coarseCoeffs = tcoarseCoeffs();
639 // Filter zone coefficients to local field
640 const labelList& za = zoneAddressing();
644 coarseCoeffs[i] = zoneCoarseCoeffs[za[i]];
647 return tcoarseCoeffs;
651 bool Foam::ggiGAMGInterface::master() const
653 return fineGgiInterface_.master();
657 bool Foam::ggiGAMGInterface::fineLevel() const
663 Foam::label Foam::ggiGAMGInterface::shadowIndex() const
665 return fineGgiInterface_.shadowIndex();
669 const Foam::ggiLduInterface& Foam::ggiGAMGInterface::shadowInterface() const
671 return refCast<const ggiLduInterface>(ldu().interfaces()[shadowIndex()]);
675 Foam::label Foam::ggiGAMGInterface::zoneSize() const
681 const Foam::labelList& Foam::ggiGAMGInterface::zoneAddressing() const
683 return zoneAddressing_;
687 const Foam::labelListList& Foam::ggiGAMGInterface::addressing() const
689 FatalErrorIn("const labelListList& ggiGAMGInterface::addressing() const")
690 << "Requested fine addressing at coarse level"
691 << abort(FatalError);
693 return labelListList::null();
697 bool Foam::ggiGAMGInterface::localParallel() const
699 return fineGgiInterface_.localParallel();
703 const Foam::scalarListList& Foam::ggiGAMGInterface::weights() const
705 FatalErrorIn("const labelListList& ggiGAMGInterface::weights() const")
706 << "Requested fine addressing at coarse level"
707 << abort(FatalError);
709 return scalarListList::null();
713 const Foam::tensorField& Foam::ggiGAMGInterface::forwardT() const
715 return fineGgiInterface_.forwardT();
719 const Foam::tensorField& Foam::ggiGAMGInterface::reverseT() const
721 return fineGgiInterface_.reverseT();
725 void Foam::ggiGAMGInterface::initTransfer
727 const Pstream::commsTypes commsType,
728 const unallocLabelList& interfaceData
731 // Label transfer is local
732 labelTransferBuffer_ = interfaceData;
736 Foam::tmp<Foam::labelField> Foam::ggiGAMGInterface::transfer
738 const Pstream::commsTypes,
739 const unallocLabelList& interfaceData
742 // Label transfer is local without global reduction
743 return this->shadowInterface().labelTransferBuffer();
747 void Foam::ggiGAMGInterface::initInternalFieldTransfer
749 const Pstream::commsTypes commsType,
750 const unallocLabelList& iF
753 // Label transfer is local without global reduction
754 labelTransferBuffer_ = interfaceInternalField(iF);
758 Foam::tmp<Foam::labelField> Foam::ggiGAMGInterface::internalFieldTransfer
760 const Pstream::commsTypes,
761 const unallocLabelList&
764 return shadowInterface().labelTransferBuffer();
768 void Foam::ggiGAMGInterface::initInternalFieldTransfer
770 const Pstream::commsTypes commsType,
771 const scalarField& iF
774 scalarField pif = interfaceInternalField(iF);
776 // New treatment. HJ, 26/Jun/2011
777 fieldTransferBuffer_ = fastReduce(pif);
781 // Expand the field, executing a reduce operation in init
782 fieldTransferBuffer_ = expand(pif);
787 Foam::tmp<Foam::scalarField> Foam::ggiGAMGInterface::internalFieldTransfer
789 const Pstream::commsTypes,
793 return shadowInterface().fieldTransferBuffer();
797 // ************************************************************************* //