1 /*---------------------------------------------------------------------------*\
3 \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
5 \\ / A nd | Copyright (C) 1991-2010 OpenCFD Ltd.
7 -------------------------------------------------------------------------------
9 This file is part of OpenFOAM.
11 OpenFOAM is free software: you can redistribute it and/or modify it
12 under the terms of the GNU General Public License as published by
13 the Free Software Foundation, either version 3 of the License, or
14 (at your option) any later version.
16 OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
28 Private member of domainDecomposition.
29 Decomposes the mesh into bits
31 \*---------------------------------------------------------------------------*/
33 #include "domainDecomposition.H"
34 #include "IOstreams.H"
35 #include "SLPtrList.H"
37 #include "primitiveMesh.H"
38 #include "cyclicPolyPatch.H"
40 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
42 void domainDecomposition::decomposeMesh(const bool filterEmptyPatches)
44 // Decide which cell goes to which processor
47 // Distribute the cells according to the given processor label
49 // calculate the addressing information for the original mesh
50 Info<< "\nCalculating original mesh data" << endl;
52 // set references to the original mesh
53 const polyBoundaryMesh& patches = boundaryMesh();
54 const faceList& fcs = faces();
55 const labelList& owner = faceOwner();
56 const labelList& neighbour = faceNeighbour();
58 // loop through the list of processor labels for the cell and add the
59 // cell shape to the list of cells for the appropriate processor
61 Info<< "\nDistributing cells to processors" << endl;
65 List<SLList<label> > procCellList(nProcs_);
67 forAll (cellToProc_, celli)
69 if (cellToProc_[celli] >= nProcs_)
71 FatalErrorIn("domainDecomposition::decomposeMesh()")
72 << "Impossible processor label " << cellToProc_[celli]
73 << "for cell " << celli
78 procCellList[cellToProc_[celli]].append(celli);
82 // Convert linked lists into normal lists
83 forAll (procCellList, procI)
85 procCellAddressing_[procI] = procCellList[procI];
89 Info << "\nDistributing faces to processors" << endl;
91 // Loop through all internal faces and decide which processor they belong to
92 // First visit all internal faces. If cells at both sides belong to the
93 // same processor, the face is an internal face. If they are different,
94 // it belongs to both processors.
98 List<SLList<label> > procFaceList(nProcs_);
100 forAll (neighbour, facei)
102 if (cellToProc_[owner[facei]] == cellToProc_[neighbour[facei]])
104 // Face internal to processor
105 procFaceList[cellToProc_[owner[facei]]].append(facei);
109 // Detect inter-processor boundaries
111 // Neighbour processor for each subdomain
112 List<SLList<label> > interProcBoundaries(nProcs_);
114 // Face labels belonging to each inter-processor boundary
115 List<SLList<SLList<label> > > interProcBFaces(nProcs_);
117 List<SLList<label> > procPatchIndex(nProcs_);
119 forAll (neighbour, facei)
121 if (cellToProc_[owner[facei]] != cellToProc_[neighbour[facei]])
123 // inter - processor patch face found. Go through the list of
124 // inside boundaries for the owner processor and try to find
125 // this inter-processor patch.
127 label ownerProc = cellToProc_[owner[facei]];
128 label neighbourProc = cellToProc_[neighbour[facei]];
130 SLList<label>::iterator curInterProcBdrsOwnIter =
131 interProcBoundaries[ownerProc].begin();
133 SLList<SLList<label> >::iterator curInterProcBFacesOwnIter =
134 interProcBFaces[ownerProc].begin();
136 bool interProcBouFound = false;
138 // WARNING: Synchronous SLList iterators
143 curInterProcBdrsOwnIter
144 != interProcBoundaries[ownerProc].end()
145 && curInterProcBFacesOwnIter
146 != interProcBFaces[ownerProc].end();
147 ++curInterProcBdrsOwnIter, ++curInterProcBFacesOwnIter
150 if (curInterProcBdrsOwnIter() == neighbourProc)
152 // the inter - processor boundary exists. Add the face
153 interProcBouFound = true;
155 curInterProcBFacesOwnIter().append(facei);
157 SLList<label>::iterator curInterProcBdrsNeiIter =
158 interProcBoundaries[neighbourProc].begin();
160 SLList<SLList<label> >::iterator
161 curInterProcBFacesNeiIter =
162 interProcBFaces[neighbourProc].begin();
164 bool neighbourFound = false;
166 // WARNING: Synchronous SLList iterators
171 curInterProcBdrsNeiIter !=
172 interProcBoundaries[neighbourProc].end()
173 && curInterProcBFacesNeiIter !=
174 interProcBFaces[neighbourProc].end();
175 ++curInterProcBdrsNeiIter,
176 ++curInterProcBFacesNeiIter
179 if (curInterProcBdrsNeiIter() == ownerProc)
181 // boundary found. Add the face
182 neighbourFound = true;
184 curInterProcBFacesNeiIter().append(facei);
187 if (neighbourFound) break;
190 if (interProcBouFound && !neighbourFound)
192 FatalErrorIn("domainDecomposition::decomposeMesh()")
193 << "Inconsistency in inter - "
194 << "processor boundary lists for processors "
195 << ownerProc << " and " << neighbourProc
196 << abort(FatalError);
200 if (interProcBouFound) break;
203 if (!interProcBouFound)
205 // inter - processor boundaries do not exist and need to
208 // set the new addressing information
211 interProcBoundaries[ownerProc].append(neighbourProc);
212 interProcBFaces[ownerProc].append(SLList<label>(facei));
215 interProcBoundaries[neighbourProc].append(ownerProc);
216 interProcBFaces[neighbourProc].append(SLList<label>(facei));
221 // Loop through patches. For cyclic boundaries detect inter-processor
222 // faces; for all other, add faces to the face list and remember start
223 // and size of all patches.
225 // for all processors, set the size of start index and patch size
226 // lists to the number of patches in the mesh
227 forAll (procPatchSize_, procI)
229 procPatchSize_[procI].setSize(patches.size());
230 procPatchStartIndex_[procI].setSize(patches.size());
233 forAll (patches, patchi)
235 // Reset size and start index for all processors
236 forAll (procPatchSize_, procI)
238 procPatchSize_[procI][patchi] = 0;
239 procPatchStartIndex_[procI][patchi] =
240 procFaceList[procI].size();
243 const label patchStart = patches[patchi].start();
245 if (!isA<cyclicPolyPatch>(patches[patchi]))
247 // Normal patch. Add faces to processor where the cell
248 // next to the face lives
250 const unallocLabelList& patchFaceCells =
251 patches[patchi].faceCells();
253 forAll (patchFaceCells, facei)
255 const label curProc = cellToProc_[patchFaceCells[facei]];
258 procFaceList[curProc].append(patchStart + facei);
260 // increment the number of faces for this patch
261 procPatchSize_[curProc][patchi]++;
266 // Cyclic patch special treatment
268 const polyPatch& cPatch = patches[patchi];
270 const label cycOffset = cPatch.size()/2;
272 // Set reference to faceCells for both patches
273 const labelList::subList firstFaceCells
279 const labelList::subList secondFaceCells
286 forAll (firstFaceCells, facei)
290 cellToProc_[firstFaceCells[facei]]
291 != cellToProc_[secondFaceCells[facei]]
294 // This face becomes an inter-processor boundary face
295 // inter - processor patch face found. Go through
296 // the list of inside boundaries for the owner
297 // processor and try to find this inter-processor
300 cyclicParallel_ = true;
302 label ownerProc = cellToProc_[firstFaceCells[facei]];
303 label neighbourProc =
304 cellToProc_[secondFaceCells[facei]];
306 SLList<label>::iterator curInterProcBdrsOwnIter =
307 interProcBoundaries[ownerProc].begin();
309 SLList<SLList<label> >::iterator
310 curInterProcBFacesOwnIter =
311 interProcBFaces[ownerProc].begin();
313 bool interProcBouFound = false;
315 // WARNING: Synchronous SLList iterators
320 curInterProcBdrsOwnIter !=
321 interProcBoundaries[ownerProc].end()
322 && curInterProcBFacesOwnIter !=
323 interProcBFaces[ownerProc].end();
324 ++curInterProcBdrsOwnIter,
325 ++curInterProcBFacesOwnIter
328 if (curInterProcBdrsOwnIter() == neighbourProc)
330 // the inter - processor boundary exists.
332 interProcBouFound = true;
334 curInterProcBFacesOwnIter().append
335 (patchStart + facei);
337 SLList<label>::iterator curInterProcBdrsNeiIter
338 = interProcBoundaries[neighbourProc].begin();
340 SLList<SLList<label> >::iterator
341 curInterProcBFacesNeiIter =
342 interProcBFaces[neighbourProc].begin();
344 bool neighbourFound = false;
346 // WARNING: Synchronous SLList iterators
351 curInterProcBdrsNeiIter
352 != interProcBoundaries[neighbourProc].end()
353 && curInterProcBFacesNeiIter
354 != interProcBFaces[neighbourProc].end();
355 ++curInterProcBdrsNeiIter,
356 ++curInterProcBFacesNeiIter
359 if (curInterProcBdrsNeiIter() == ownerProc)
361 // boundary found. Add the face
362 neighbourFound = true;
364 curInterProcBFacesNeiIter()
373 if (neighbourFound) break;
376 if (interProcBouFound && !neighbourFound)
380 "domainDecomposition::decomposeMesh()"
381 ) << "Inconsistency in inter-processor "
382 << "boundary lists for processors "
383 << ownerProc << " and " << neighbourProc
384 << " in cyclic boundary matching"
385 << abort(FatalError);
389 if (interProcBouFound) break;
392 if (!interProcBouFound)
394 // inter - processor boundaries do not exist
395 // and need to be created
397 // set the new addressing information
400 interProcBoundaries[ownerProc]
401 .append(neighbourProc);
402 interProcBFaces[ownerProc]
403 .append(SLList<label>(patchStart + facei));
406 interProcBoundaries[neighbourProc]
408 interProcBFaces[neighbourProc]
422 // This cyclic face remains on the processor
423 label ownerProc = cellToProc_[firstFaceCells[facei]];
426 procFaceList[ownerProc].append(patchStart + facei);
428 // increment the number of faces for this patch
429 procPatchSize_[ownerProc][patchi]++;
431 // Note: I cannot add the other side of the cyclic
432 // boundary here because this would violate the order.
433 // They will be added in a separate loop below
438 // Ordering in cyclic boundaries is important.
439 // Add the other half of cyclic faces for cyclic boundaries
440 // that remain on the processor
441 forAll (secondFaceCells, facei)
445 cellToProc_[firstFaceCells[facei]]
446 == cellToProc_[secondFaceCells[facei]]
449 // This cyclic face remains on the processor
450 label ownerProc = cellToProc_[firstFaceCells[facei]];
452 // add the second face
453 procFaceList[ownerProc].append
454 (patchStart + cycOffset + facei);
456 // increment the number of faces for this patch
457 procPatchSize_[ownerProc][patchi]++;
463 // Convert linked lists into normal lists
464 // Add inter-processor boundaries and remember start indices
465 forAll (procFaceList, procI)
467 // Get internal and regular boundary processor faces
468 SLList<label>& curProcFaces = procFaceList[procI];
470 // Get reference to processor face addressing
471 labelList& curProcFaceAddressing = procFaceAddressing_[procI];
473 labelList& curProcNeighbourProcessors =
474 procNeighbourProcessors_[procI];
476 labelList& curProcProcessorPatchSize =
477 procProcessorPatchSize_[procI];
479 labelList& curProcProcessorPatchStartIndex =
480 procProcessorPatchStartIndex_[procI];
482 // calculate the size
483 label nFacesOnProcessor = curProcFaces.size();
487 SLList<SLList<label> >::iterator curInterProcBFacesIter =
488 interProcBFaces[procI].begin();
489 curInterProcBFacesIter != interProcBFaces[procI].end();
490 ++curInterProcBFacesIter
493 nFacesOnProcessor += curInterProcBFacesIter().size();
496 curProcFaceAddressing.setSize(nFacesOnProcessor);
498 // Fill in the list. Calculate turning index.
499 // Turning index will be -1 only for some faces on processor
500 // boundaries, i.e. the ones where the current processor ID
501 // is in the cell which is a face neighbour.
502 // Turning index is stored as the sign of the face addressing list
506 // Add internal and boundary faces
507 // Remember to increment the index by one such that the
508 // turning index works properly.
511 SLList<label>::iterator curProcFacesIter = curProcFaces.begin();
512 curProcFacesIter != curProcFaces.end();
516 curProcFaceAddressing[nFaces] = curProcFacesIter() + 1;
520 // Add inter-processor boundary faces. At the beginning of each
521 // patch, grab the patch start index and size
523 curProcNeighbourProcessors.setSize
525 interProcBoundaries[procI].size()
528 curProcProcessorPatchSize.setSize
530 interProcBoundaries[procI].size()
533 curProcProcessorPatchStartIndex.setSize
535 interProcBoundaries[procI].size()
538 label nProcPatches = 0;
540 SLList<label>::iterator curInterProcBdrsIter =
541 interProcBoundaries[procI].begin();
543 SLList<SLList<label> >::iterator curInterProcBFacesIter =
544 interProcBFaces[procI].begin();
549 curInterProcBdrsIter != interProcBoundaries[procI].end()
550 && curInterProcBFacesIter != interProcBFaces[procI].end();
551 ++curInterProcBdrsIter, ++curInterProcBFacesIter
554 curProcNeighbourProcessors[nProcPatches] =
555 curInterProcBdrsIter();
557 // Get start index for processor patch
558 curProcProcessorPatchStartIndex[nProcPatches] = nFaces;
561 curProcProcessorPatchSize[nProcPatches];
565 // add faces for this processor boundary
569 SLList<label>::iterator curFacesIter =
570 curInterProcBFacesIter().begin();
571 curFacesIter != curInterProcBFacesIter().end();
577 // Remember to increment the index by one such that the
578 // turning index works properly.
579 if (cellToProc_[owner[curFacesIter()]] == procI)
581 curProcFaceAddressing[nFaces] = curFacesIter() + 1;
586 curProcFaceAddressing[nFaces] = -(curFacesIter() + 1);
589 // increment the size
600 Info << "\nCalculating processor boundary addressing" << endl;
601 // For every patch of processor boundary, find the index of the original
602 // patch. Mis-alignment is caused by the fact that patches with zero size
603 // are omitted. For processor patches, set index to -1.
604 // At the same time, filter the procPatchSize_ and procPatchStartIndex_
605 // lists to exclude zero-size patches
606 forAll (procPatchSize_, procI)
608 // Make a local copy of old lists
609 const labelList oldPatchSizes = procPatchSize_[procI];
611 const labelList oldPatchStarts = procPatchStartIndex_[procI];
613 labelList& curPatchSizes = procPatchSize_[procI];
615 labelList& curPatchStarts = procPatchStartIndex_[procI];
617 const labelList& curProcessorPatchSizes =
618 procProcessorPatchSize_[procI];
620 labelList& curBoundaryAddressing = procBoundaryAddressing_[procI];
622 curBoundaryAddressing.setSize
625 + curProcessorPatchSizes.size()
630 forAll (oldPatchSizes, patchi)
632 if (!filterEmptyPatches || oldPatchSizes[patchi] > 0)
634 curBoundaryAddressing[nPatches] = patchi;
636 curPatchSizes[nPatches] = oldPatchSizes[patchi];
638 curPatchStarts[nPatches] = oldPatchStarts[patchi];
644 // reset to the size of live patches
645 curPatchSizes.setSize(nPatches);
646 curPatchStarts.setSize(nPatches);
648 forAll (curProcessorPatchSizes, procPatchI)
650 curBoundaryAddressing[nPatches] = -1;
655 curBoundaryAddressing.setSize(nPatches);
658 Info << "\nDistributing points to processors" << endl;
659 // For every processor, loop through the list of faces for the processor.
660 // For every face, loop through the list of points and mark the point as
661 // used for the processor. Collect the list of used points for the
664 forAll (procPointAddressing_, procI)
666 boolList pointLabels(nPoints(), false);
668 // Get reference to list of used faces
669 const labelList& procFaceLabels = procFaceAddressing_[procI];
671 forAll (procFaceLabels, facei)
673 // Because of the turning index, some labels may be negative
674 const labelList& facePoints = fcs[mag(procFaceLabels[facei]) - 1];
676 forAll (facePoints, pointi)
678 // Mark the point as used
679 pointLabels[facePoints[pointi]] = true;
683 // Collect the used points
684 labelList& procPointLabels = procPointAddressing_[procI];
686 procPointLabels.setSize(pointLabels.size());
688 label nUsedPoints = 0;
690 forAll (pointLabels, pointi)
692 if (pointLabels[pointi])
694 procPointLabels[nUsedPoints] = pointi;
700 // Reset the size of used points
701 procPointLabels.setSize(nUsedPoints);
704 // Gather data about globally shared points
708 labelList pointsUsage(nPoints(), 0);
710 // Globally shared points are the ones used by more than 2 processors
711 // Size the list approximately and gather the points
712 labelHashSet gSharedPoints
714 min(100, nPoints()/1000)
717 // Loop through all the processors and mark up points used by
718 // processor boundaries. When a point is used twice, it is a
719 // globally shared point
721 for (label procI = 0; procI < nProcs_; procI++)
723 // Get list of face labels
724 const labelList& curFaceLabels = procFaceAddressing_[procI];
726 // Get start of processor faces
727 const labelList& curProcessorPatchStarts =
728 procProcessorPatchStartIndex_[procI];
730 const labelList& curProcessorPatchSizes =
731 procProcessorPatchSize_[procI];
733 // Reset the lookup list
736 forAll (curProcessorPatchStarts, patchi)
738 const label curStart = curProcessorPatchStarts[patchi];
739 const label curEnd = curStart + curProcessorPatchSizes[patchi];
743 label facei = curStart;
748 // Mark the original face as used
749 // Remember to decrement the index by one (turning index)
751 const label curF = mag(curFaceLabels[facei]) - 1;
753 const face& f = fcs[curF];
757 if (pointsUsage[f[pointi]] == 0)
759 // Point not previously used
760 pointsUsage[f[pointi]] = patchi + 1;
762 else if (pointsUsage[f[pointi]] != patchi + 1)
764 // Point used by some other patch = global point!
765 gSharedPoints.insert(f[pointi]);
772 // Grab the result from the hash list
773 globallySharedPoints_ = gSharedPoints.toc();
774 sort(globallySharedPoints_);