Merge branch 'master' of ssh://git.code.sf.net/p/foam-extend/foam-extend-3.2
[foam-extend-3.2.git] / src / foam / matrices / lduMatrix / solvers / GAMG / interfaces / ggiGAMGInterface / ggiGAMGInterface.C
bloba3dd0485982e9a02c9e250bbe47a50261ad36917
1 /*---------------------------------------------------------------------------*\
2   =========                 |
3   \\      /  F ield         | foam-extend: Open Source CFD
4    \\    /   O peration     | Version:     3.2
5     \\  /    A nd           | Web:         http://www.foam-extend.org
6      \\/     M anipulation  | For copyright notice see file Copyright
7 -------------------------------------------------------------------------------
8 License
9     This file is part of foam-extend.
11     foam-extend is free software: you can redistribute it and/or modify it
12     under the terms of the GNU General Public License as published by the
13     Free Software Foundation, either version 3 of the License, or (at your
14     option) any later version.
16     foam-extend is distributed in the hope that it will be useful, but
17     WITHOUT ANY WARRANTY; without even the implied warranty of
18     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
19     General Public License for more details.
21     You should have received a copy of the GNU General Public License
22     along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
24 \*---------------------------------------------------------------------------*/
26 #include "ggiGAMGInterface.H"
27 #include "addToRunTimeSelectionTable.H"
29 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
31 namespace Foam
33     defineTypeNameAndDebug(ggiGAMGInterface, 0);
34     addToRunTimeSelectionTable
35     (
36         GAMGInterface,
37         ggiGAMGInterface,
38         lduInterface
39     );
43 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
45 void Foam::ggiGAMGInterface::initFastReduce() const
47     // Init should be executed only once
48     initReduce_ = true;
50     // Establish parallel comms pattern
51     if (!localParallel() && Pstream::parRun())
52     {
53         // Only master handles communication
54         if (Pstream::master())
55         {
56             receiveAddr_.setSize(Pstream::nProcs());
57             sendAddr_.setSize(Pstream::nProcs());
59             receiveAddr_[0] = zoneAddressing();
61             for (label procI = 1; procI < Pstream::nProcs(); procI++)
62             {
63                 // Note: must use normal comms because the size of the
64                 // communicated lists is unknown on the receiving side
65                 // HJ, 4/Jun/2011
67                 // Opt: reconsider mode of communication
68                 IPstream ip(Pstream::scheduled, procI);
70                 receiveAddr_[procI] = labelList(ip);
72                 sendAddr_[procI] = labelList(ip);
73             }
74         }
75         else
76         {
77             // Opt: reconsider mode of communication
78             OPstream op(Pstream::scheduled, Pstream::masterNo());
80             op << zoneAddressing() << shadowInterface().zoneAddressing();
81         }
82     }
86 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
88 Foam::ggiGAMGInterface::ggiGAMGInterface
90     const lduPrimitiveMesh& lduMesh,
91     const lduInterface& fineInterface,
92     const labelField& localRestrictAddressing,
93     const labelField& neighbourRestrictAddressing
96     GAMGInterface(lduMesh),
97     fineGgiInterface_(refCast<const ggiLduInterface>(fineInterface)),
98     zoneSize_(0),
99     zoneAddressing_(),
100     initReduce_(false),
101     receiveAddr_(),
102     sendAddr_()
104     // Note.
105     // All processors will do the same coarsening and then filter
106     // the addressing to the local processor
107     // HJ, 1/Apr/2009
109     // Note.
110     // Signalling in global clustering requires me to recognise clustering
111     // from separate processors as separate.  In the first phase, this will be
112     // used to recognise cluster from each processor as separate and in the
113     // second phase it will be used to filter local processor faces from
114     // the global patch.
115     // Currently, I am calculating unique cluster index as:
116     //
117     // id = cluster + procOffset*myProcID
118     // With procOffset = 1 million, this should be sufficient for 2000 CPUs
119     // with 2 million coarse cells each.  For larger numbers, I need a
120     // larger max int, which can be changed on request
121     // HJ, 1/Apr/2009
123     // Expand the local and neighbour addressing to full zone size
124     labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0);
126     // Memory management, local
127     {
128         const labelList& addr = fineGgiInterface_.zoneAddressing();
130         forAll (addr, i)
131         {
132             localExpandAddressing[addr[i]] =
133                 localRestrictAddressing[i] + procOffset*Pstream::myProcNo();
134         }
136         if (!localParallel())
137         {
138             reduce(localExpandAddressing, sumOp<labelField>());
139         }
140     }
142     labelField neighbourExpandAddressing
143     (
144         fineGgiInterface_.shadowInterface().zoneSize(),
145         0
146     );
148     // Memory management, neighbour
149     {
150         const labelList& addr =
151             fineGgiInterface_.shadowInterface().zoneAddressing();
153         forAll (addr, i)
154         {
155             neighbourExpandAddressing[addr[i]] =
156                 neighbourRestrictAddressing[i]
157               + procOffset*Pstream::myProcNo();
158         }
160         if (!localParallel())
161         {
162             reduce(neighbourExpandAddressing, sumOp<labelField>());
163         }
164     }
166     // Make a lookup table of entries for owner/neighbour.
167     // All sizes are guessed at the size of fine interface
168     // HJ, 19/Feb/2009
170     HashTable<SLList<label>, label, Hash<label> > neighboursTable
171     (
172         localExpandAddressing.size()
173     );
175     // Table of face-sets to be agglomerated
176     HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
177     (
178         localExpandAddressing.size()
179     );
181     // Table of face-sets weights to be agglomerated
182     HashTable<SLList<SLList<scalar> >, label, Hash<label> >
183         faceFaceWeightsTable
184         (
185             localExpandAddressing.size()
186         );
188     // Count the number of coarse faces
189     label nCoarseFaces = 0;
191     // Count the number of agglomeration pairs
192     label nAgglomPairs = 0;
194     // On the fine level, addressing is made in a labelListList
195     if (fineGgiInterface_.fineLevel())
196     {
197         const labelListList& fineAddr = fineGgiInterface_.addressing();
198         const scalarListList& fineWeights = fineGgiInterface_.weights();
200         forAll (fineAddr, ffI)
201         {
202             const labelList& curFineNbrs = fineAddr[ffI];
203             const scalarList& curFineWeigts = fineWeights[ffI];
205             forAll (curFineNbrs, nbrI)
206             {
207                 label curMaster = -1;
208                 label curSlave = -1;
210                 // My label = ffI
211                 // Nbr label = nnI
212                 const label nnI = curFineNbrs[nbrI];
213                 const scalar curNW = curFineWeigts[nbrI];
215                 if (fineGgiInterface_.master())
216                 {
217                     // Master side
218                     curMaster = localExpandAddressing[ffI];
219                     curSlave = neighbourExpandAddressing[nnI];
220                 }
221                 else
222                 {
223                     // Slave side
224                     curMaster = neighbourExpandAddressing[nnI];
225                     curSlave = localExpandAddressing[ffI];
226                 }
228                 // Look for the master cell.  If it has already got a face,
229                 // add the coefficient to the face.  If not, create a new
230                 // face.
231                 if (neighboursTable.found(curMaster))
232                 {
233                     // Check all current neighbours to see if the current
234                     // slave already exists.  If so, add the coefficient.
236                     SLList<label>& curNbrs = neighboursTable.find(curMaster)();
238                     SLList<SLList<label> >& curFaceFaces =
239                         faceFaceTable.find(curMaster)();
241                     SLList<SLList<scalar> >& curFaceWeights =
242                         faceFaceWeightsTable.find(curMaster)();
244                     bool nbrFound = false;
246                     SLList<label>::iterator nbrsIter = curNbrs.begin();
248                     SLList<SLList<label> >::iterator faceFacesIter =
249                         curFaceFaces.begin();
251                     SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
252                         curFaceWeights.begin();
254                     for
255                     (
256                         ;
257                         nbrsIter != curNbrs.end()
258                      && faceFacesIter != curFaceFaces.end()
259                      && faceFaceWeightsIter != curFaceWeights.end();
260                         ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
261                     )
262                     {
263                         // Check neighbour slave
264                         if (nbrsIter() == curSlave)
265                         {
266                             nbrFound = true;
267                             faceFacesIter().append(ffI);
268                             faceFaceWeightsIter().append(curNW);
269                             nAgglomPairs++;
271                             break;
272                         }
273                     }
275                     if (!nbrFound)
276                     {
277                         curNbrs.append(curSlave);
278                         curFaceFaces.append(SLList<label>(ffI));
279                         curFaceWeights.append(SLList<scalar>(curNW));
281                         // New coarse face created
282                         nCoarseFaces++;
283                         nAgglomPairs++;
284                     }
285                 }
286                 else
287                 {
288                     // This master has got no neighbours yet.  Add a neighbour
289                     // and a coefficient, thus creating a new face
290                     neighboursTable.insert(curMaster, SLList<label>(curSlave));
292                     faceFaceTable.insert
293                     (
294                         curMaster,
295                         SLList<SLList<label> >(SLList<label>(ffI))
296                     );
298                     faceFaceWeightsTable.insert
299                     (
300                         curMaster,
301                         SLList<SLList<scalar> >(SLList<scalar>(curNW))
302                     );
304                     // New coarse face created
305                     nCoarseFaces++;
306                     nAgglomPairs++;
307                 }
308             } // end for all current neighbours
309         } // end for all fine faces
310     }
311     else
312     {
313         // Coarse level, addressing is stored in faceCells
314         forAll (localExpandAddressing, ffi)
315         {
316             label curMaster = -1;
317             label curSlave = -1;
319             // Do switching on master/slave indices based on the
320             // owner/neighbour of the processor index such that
321             // both sides get the same answer.
322             if (master())
323             {
324                 // Master side
325                 curMaster = localExpandAddressing[ffi];
326                 curSlave = neighbourExpandAddressing[ffi];
327             }
328             else
329             {
330                 // Slave side
331                 curMaster = neighbourExpandAddressing[ffi];
332                 curSlave = localExpandAddressing[ffi];
333             }
335             // Look for the master cell.  If it has already got a face,
336             // add the coefficient to the face.  If not, create a new face.
337             if (neighboursTable.found(curMaster))
338             {
339                 // Check all current neighbours to see if the current slave
340                 // already exists and if so, add the fine face
341                 // to the agglomeration.
343                 SLList<label>& curNbrs = neighboursTable.find(curMaster)();
345                 SLList<SLList<label> >& curFaceFaces =
346                     faceFaceTable.find(curMaster)();
348                     SLList<SLList<scalar> >& curFaceWeights =
349                         faceFaceWeightsTable.find(curMaster)();
351                 bool nbrFound = false;
353                 SLList<label>::iterator nbrsIter = curNbrs.begin();
355                 SLList<SLList<label> >::iterator faceFacesIter =
356                     curFaceFaces.begin();
358                 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
359                     curFaceWeights.begin();
361                 for
362                 (
363                     ;
364                     nbrsIter != curNbrs.end()
365                  && faceFacesIter != curFaceFaces.end()
366                  && faceFaceWeightsIter != curFaceWeights.end();
367                     ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
368                 )
369                 {
370                     // Check neighbour slave
371                     if (nbrsIter() == curSlave)
372                     {
373                         nbrFound = true;
374                         faceFacesIter().append(ffi);
375                         // Add dummy weight
376                         faceFaceWeightsIter().append(1.0);
377                         nAgglomPairs++;
378                         break;
379                     }
380                 }
382                 if (!nbrFound)
383                 {
384                     curNbrs.append(curSlave);
385                     curFaceFaces.append(SLList<label>(ffi));
386                     // Add dummy weight
387                     curFaceWeights.append(SLList<scalar>(1.0));
389                     // New coarse face created
390                     nCoarseFaces++;
391                     nAgglomPairs++;
392                 }
393             }
394             else
395             {
396                 // This master has got no neighbours yet.  Add a neighbour
397                 // and a coefficient, thus creating a new face
398                 neighboursTable.insert(curMaster, SLList<label>(curSlave));
400                 faceFaceTable.insert
401                 (
402                     curMaster,
403                     SLList<SLList<label> >(SLList<label>(ffi))
404                 );
406                 // Add dummy weight
407                 faceFaceWeightsTable.insert
408                 (
409                     curMaster,
410                     SLList<SLList<scalar> >(SLList<scalar>(1.0))
411                 );
413                 // New coarse face created
414                 nCoarseFaces++;
415                 nAgglomPairs++;
416             }
417         } // end for all fine faces
418     }
420     faceCells_.setSize(nCoarseFaces, -1);
421     fineAddressing_.setSize(nAgglomPairs, -1);
422     restrictAddressing_.setSize(nAgglomPairs, -1);
423     restrictWeights_.setSize(nAgglomPairs);
425     labelList contents = neighboursTable.toc();
427     // Sort makes sure the order is identical on both sides.
428     // HJ, 20/Feb.2009
429     sort(contents);
431     // Grab zone size and create zone addressing
432     zoneSize_ = nCoarseFaces;
434     zoneAddressing_.setSize(nCoarseFaces);
435     label nProcFaces = 0;
437     // Reset face counter for re-use
438     nCoarseFaces = 0;
439     nAgglomPairs = 0;
441     if (master())
442     {
443         // On master side, the owner addressing is stored in table of contents
444         forAll (contents, masterI)
445         {
446             SLList<label>& curNbrs =
447                 neighboursTable.find(contents[masterI])();
449             // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
451             SLList<SLList<label> >& curFaceFaces =
452                 faceFaceTable.find(contents[masterI])();
454             SLList<SLList<scalar> >& curFaceWeights =
455                 faceFaceWeightsTable.find(contents[masterI])();
457             SLList<label>::iterator nbrsIter = curNbrs.begin();
458             SLList<SLList<label> >::iterator faceFacesIter =
459                 curFaceFaces.begin();
461             SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
462                 curFaceWeights.begin();
464             for
465             (
466                 ;
467                 nbrsIter != curNbrs.end()
468              && faceFacesIter != curFaceFaces.end()
469              && faceFaceWeightsIter != curFaceWeights.end();
470                 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
471             )
472             {
473                 // Check if master is on local processor
474                 if
475                 (
476                     contents[masterI] >= procOffset*Pstream::myProcNo()
477                  && contents[masterI] < procOffset*(Pstream::myProcNo() + 1)
478                 )
479                 {
480                     // Record that this face belongs locally
481                     zoneAddressing_[nProcFaces] = nCoarseFaces;
482                     faceCells_[nProcFaces] =
483                         contents[masterI] - procOffset*Pstream::myProcNo();
484                     nProcFaces++;
486                     SLList<label>::iterator facesIter =
487                         faceFacesIter().begin();
488                     SLList<scalar>::iterator weightsIter =
489                         faceFaceWeightsIter().begin();
491                     for
492                     (
493                         ;
494                         facesIter != faceFacesIter().end()
495                      && weightsIter != faceFaceWeightsIter().end();
496                         ++facesIter, ++weightsIter
497                     )
498                     {
499                         fineAddressing_[nAgglomPairs] = facesIter();
500                         restrictAddressing_[nAgglomPairs] = nCoarseFaces;
501                         restrictWeights_[nAgglomPairs] = weightsIter();
502                         nAgglomPairs++;
503                     }
504                 }
506                 // Not a local face, but still created in global zone
507                 nCoarseFaces++;
508             }
509         }
511         // Resize arrays: not all of ggi is used locally
512         faceCells_.setSize(nProcFaces);
513         zoneAddressing_.setSize(nProcFaces);
515         fineAddressing_.setSize(nAgglomPairs);
516         restrictAddressing_.setSize(nAgglomPairs);
517         restrictWeights_.setSize(nAgglomPairs);
518     }
519     else
520     {
521         // On slave side, the owner addressing is stored in linked lists
522         forAll (contents, masterI)
523         {
524             // Note: master processor index is irrelevant.  HJ, 1/Apr/2009
526             SLList<label>& curNbrs = neighboursTable.find(contents[masterI])();
528             SLList<SLList<label> >& curFaceFaces =
529                 faceFaceTable.find(contents[masterI])();
531             SLList<SLList<scalar> >& curFaceWeights =
532                 faceFaceWeightsTable.find(contents[masterI])();
534             SLList<label>::iterator nbrsIter = curNbrs.begin();
536             SLList<SLList<label> >::iterator faceFacesIter =
537                 curFaceFaces.begin();
539             SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
540                 curFaceWeights.begin();
541             for
542             (
543                 ;
544                 nbrsIter != curNbrs.end()
545              && faceFacesIter != curFaceFaces.end()
546              && faceFaceWeightsIter != curFaceWeights.end();
547                 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
548             )
549             {
550                 // Check if the face is on local processor
551                 if
552                 (
553                     nbrsIter() >= procOffset*Pstream::myProcNo()
554                  && nbrsIter() < procOffset*(Pstream::myProcNo() + 1)
555                 )
556                 {
557                     // Record that this face belongs locally.
558                     zoneAddressing_[nProcFaces] = nCoarseFaces;
559                     faceCells_[nProcFaces] =
560                         nbrsIter() - procOffset*Pstream::myProcNo();
561                     nProcFaces++;
563                     SLList<label>::iterator facesIter =
564                         faceFacesIter().begin();
566                     SLList<scalar>::iterator weightsIter =
567                         faceFaceWeightsIter().begin();
569                     for
570                     (
571                         ;
572                         facesIter != faceFacesIter().end()
573                      && weightsIter != faceFaceWeightsIter().end();
574                         ++facesIter, ++weightsIter
575                     )
576                     {
577                         fineAddressing_[nAgglomPairs] = facesIter();
578                         restrictAddressing_[nAgglomPairs] = nCoarseFaces;
579                         restrictWeights_[nAgglomPairs] = weightsIter();
580                         nAgglomPairs++;
581                     }
582                 }
584                 nCoarseFaces++;
585             }
586         }
588         // Resize arrays: not all of ggi is used locally
589         faceCells_.setSize(nProcFaces);
590         zoneAddressing_.setSize(nProcFaces);
591         fineAddressing_.setSize(nAgglomPairs);
592         restrictAddressing_.setSize(nAgglomPairs);
593         restrictWeights_.setSize(nAgglomPairs);
594     }
598 // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
600 Foam::ggiGAMGInterface::~ggiGAMGInterface()
604 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
606 Foam::tmp<Foam::scalarField> Foam::ggiGAMGInterface::agglomerateCoeffs
608     const scalarField& fineCoeffs
609 ) const
611     // Reassemble fine coefficients to full fine zone size
612     scalarField zoneFineCoeffs(fineGgiInterface_.zoneSize(), 0);
614     const labelList& fineZa = fineGgiInterface_.zoneAddressing();
616     forAll (fineZa, i)
617     {
618         zoneFineCoeffs[fineZa[i]] = fineCoeffs[i];
619     }
621     // Reduce zone data
622     if (!localParallel())
623     {
624         reduce(zoneFineCoeffs, sumOp<scalarField>());
625     }
627     scalarField zoneCoarseCoeffs(zoneSize(), 0);
629     // Restrict coefficient
630     forAll(restrictAddressing_, ffi)
631     {
632         zoneCoarseCoeffs[restrictAddressing_[ffi]] +=
633             restrictWeights_[ffi]*zoneFineCoeffs[fineAddressing_[ffi]];
634     }
636     tmp<scalarField> tcoarseCoeffs(new scalarField(size(), 0.0));
637     scalarField& coarseCoeffs = tcoarseCoeffs();
639     // Filter zone coefficients to local field
640     const labelList& za = zoneAddressing();
642     forAll (za, i)
643     {
644         coarseCoeffs[i] = zoneCoarseCoeffs[za[i]];
645     }
647     return tcoarseCoeffs;
651 bool Foam::ggiGAMGInterface::master() const
653     return fineGgiInterface_.master();
657 bool Foam::ggiGAMGInterface::fineLevel() const
659     return false;
663 Foam::label Foam::ggiGAMGInterface::shadowIndex() const
665     return fineGgiInterface_.shadowIndex();
669 const Foam::ggiLduInterface& Foam::ggiGAMGInterface::shadowInterface() const
671     return refCast<const ggiLduInterface>(ldu().interfaces()[shadowIndex()]);
675 Foam::label Foam::ggiGAMGInterface::zoneSize() const
677     return zoneSize_;
681 const Foam::labelList& Foam::ggiGAMGInterface::zoneAddressing() const
683     return zoneAddressing_;
687 const Foam::labelListList& Foam::ggiGAMGInterface::addressing() const
689     FatalErrorIn("const labelListList& ggiGAMGInterface::addressing() const")
690         << "Requested fine addressing at coarse level"
691         << abort(FatalError);
693     return labelListList::null();
697 bool Foam::ggiGAMGInterface::localParallel() const
699     return fineGgiInterface_.localParallel();
703 const Foam::scalarListList& Foam::ggiGAMGInterface::weights() const
705     FatalErrorIn("const labelListList& ggiGAMGInterface::weights() const")
706         << "Requested fine addressing at coarse level"
707         << abort(FatalError);
709     return scalarListList::null();
713 const Foam::tensorField& Foam::ggiGAMGInterface::forwardT() const
715     return fineGgiInterface_.forwardT();
719 const Foam::tensorField& Foam::ggiGAMGInterface::reverseT() const
721     return fineGgiInterface_.reverseT();
725 void Foam::ggiGAMGInterface::initTransfer
727     const Pstream::commsTypes commsType,
728     const unallocLabelList& interfaceData
729 ) const
731     // Label transfer is local
732     labelTransferBuffer_ = interfaceData;
736 Foam::tmp<Foam::labelField> Foam::ggiGAMGInterface::transfer
738     const Pstream::commsTypes,
739     const unallocLabelList& interfaceData
740 ) const
742     // Label transfer is local without global reduction
743     return this->shadowInterface().labelTransferBuffer();
747 void Foam::ggiGAMGInterface::initInternalFieldTransfer
749     const Pstream::commsTypes commsType,
750     const unallocLabelList& iF
751 ) const
753     // Label transfer is local without global reduction
754     labelTransferBuffer_ = interfaceInternalField(iF);
758 Foam::tmp<Foam::labelField> Foam::ggiGAMGInterface::internalFieldTransfer
760     const Pstream::commsTypes,
761     const unallocLabelList&
762 ) const
764     return shadowInterface().labelTransferBuffer();
768 void Foam::ggiGAMGInterface::initInternalFieldTransfer
770     const Pstream::commsTypes commsType,
771     const scalarField& iF
772 ) const
774     scalarField pif = interfaceInternalField(iF);
776     // New treatment.  HJ, 26/Jun/2011
777     fieldTransferBuffer_ = fastReduce(pif);
779     // Old treatment
780 #if(0)
781     // Expand the field, executing a reduce operation in init
782     fieldTransferBuffer_ = expand(pif);
783 #endif
787 Foam::tmp<Foam::scalarField> Foam::ggiGAMGInterface::internalFieldTransfer
789     const Pstream::commsTypes,
790     const scalarField&
791 ) const
793     return shadowInterface().fieldTransferBuffer();
797 // ************************************************************************* //