src/gromacs/ewald/pme-gpu-3dfft-ocl.cpp

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2016,2017,2018, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35
  36 /*! \internal \file
  37  *  \brief Implements OpenCL 3D FFT routines for PME GPU.
  38  *
  39  *  \author Aleksei Iupinov <a.yupinov@gmail.com>
  40  *  \ingroup module_ewald
  41  */
  42
  43 #include "gmxpre.h"
  44
  45 #include <array>
  46
  47 #include "gromacs/utility/exceptions.h"
  48 #include "gromacs/utility/gmxassert.h"
  49 #include "gromacs/utility/stringutil.h"
  50
  51 #include "pme-gpu-3dfft.h"
  52 #include "pme-gpu-internal.h"
  53 #include "pme-gpu-types.h"
  54 #include "pme-gpu-types-host-impl.h"
  55
  56 //! Throws the exception on clFFT error
  57 static void handleClfftError(clfftStatus status, const char *msg)
  58 {
  59     // Supposedly it's just a superset of standard OpenCL errors
  60     if (status != CLFFT_SUCCESS)
  61     {
  62         GMX_THROW(gmx::InternalError(gmx::formatString("%s: %d", msg, status)));
  63     }
  64 }
  65
  66 GpuParallel3dFft::GpuParallel3dFft(const PmeGpu *pmeGpu)
  67 {
  68     // Extracting all the data from PME GPU
  69     std::array<size_t, DIM> realGridSize, realGridSizePadded, complexGridSizePadded;
  70
  71     GMX_RELEASE_ASSERT(!pme_gpu_uses_dd(pmeGpu), "FFT decomposition not implemented");
  72     PmeGpuKernelParamsBase *kernelParamsPtr = pmeGpu->kernelParams.get();
  73     for (int i = 0; i < DIM; i++)
  74     {
  75         realGridSize[i]          = kernelParamsPtr->grid.realGridSize[i];
  76         realGridSizePadded[i]    = kernelParamsPtr->grid.realGridSizePadded[i];
  77         complexGridSizePadded[i] = kernelParamsPtr->grid.complexGridSizePadded[i];
  78         GMX_ASSERT(kernelParamsPtr->grid.complexGridSizePadded[i] == kernelParamsPtr->grid.complexGridSize[i], "Complex padding not implemented");
  79     }
  80     cl_context context = pmeGpu->archSpecific->context;
  81     commandStreams_.push_back(pmeGpu->archSpecific->pmeStream);
  82     realGrid_    = kernelParamsPtr->grid.d_realGrid;
  83     complexGrid_ = kernelParamsPtr->grid.d_fourierGrid;
  84     const bool performOutOfPlaceFFT = pmeGpu->archSpecific->performOutOfPlaceFFT;
  85
  86
  87     // clFFT expects row-major, so dimensions/strides are reversed (ZYX instead of XYZ)
  88     std::array<size_t, DIM> realGridDimensions = {
  89         realGridSize[ZZ],
  90         realGridSize[YY],
  91         realGridSize[XX]
  92     };
  93     std::array<size_t, DIM> realGridStrides = {
  94         1,
  95         realGridSizePadded[ZZ],
  96         realGridSizePadded[YY] * realGridSizePadded[ZZ]
  97     };
  98     std::array<size_t, DIM> complexGridStrides = {
  99         1,
 100         complexGridSizePadded[ZZ],
 101         complexGridSizePadded[YY] * complexGridSizePadded[ZZ]
 102     };
 103
 104     constexpr clfftDim      dims = CLFFT_3D;
 105     handleClfftError(clfftCreateDefaultPlan(&planR2C_, context, dims, realGridDimensions.data()), "clFFT planning failure");
 106     handleClfftError(clfftSetResultLocation(planR2C_, performOutOfPlaceFFT ? CLFFT_OUTOFPLACE : CLFFT_INPLACE), "clFFT planning failure");
 107     handleClfftError(clfftSetPlanPrecision(planR2C_, CLFFT_SINGLE), "clFFT planning failure");
 108     constexpr cl_float scale = 1.0;
 109     handleClfftError(clfftSetPlanScale(planR2C_, CLFFT_FORWARD, scale), "clFFT coefficient setup failure");
 110     handleClfftError(clfftSetPlanScale(planR2C_, CLFFT_BACKWARD, scale), "clFFT coefficient setup failure");
 111
 112     // The only difference between 2 plans is direction
 113     handleClfftError(clfftCopyPlan(&planC2R_, context, planR2C_), "clFFT plan copying failure");
 114
 115     handleClfftError(clfftSetLayout(planR2C_, CLFFT_REAL, CLFFT_HERMITIAN_INTERLEAVED), "clFFT R2C layout failure");
 116     handleClfftError(clfftSetLayout(planC2R_, CLFFT_HERMITIAN_INTERLEAVED, CLFFT_REAL), "clFFT C2R layout failure");
 117
 118     handleClfftError(clfftSetPlanInStride(planR2C_, dims, realGridStrides.data()), "clFFT stride setting failure");
 119     handleClfftError(clfftSetPlanOutStride(planR2C_, dims, complexGridStrides.data()), "clFFT stride setting failure");
 120
 121     handleClfftError(clfftSetPlanInStride(planC2R_, dims, complexGridStrides.data()), "clFFT stride setting failure");
 122     handleClfftError(clfftSetPlanOutStride(planC2R_, dims, realGridStrides.data()), "clFFT stride setting failure");
 123
 124     handleClfftError(clfftBakePlan(planR2C_, commandStreams_.size(), commandStreams_.data(), nullptr, nullptr), "clFFT precompiling failure");
 125     handleClfftError(clfftBakePlan(planC2R_, commandStreams_.size(), commandStreams_.data(), nullptr, nullptr), "clFFT precompiling failure");
 126
 127     //TODO: implement solve kernel as R2C FFT callback
 128     //TODO: disable last transpose (clfftSetPlanTransposeResult)
 129 }
 130
 131 GpuParallel3dFft::~GpuParallel3dFft()
 132 {
 133     clfftDestroyPlan(&planR2C_);
 134     clfftDestroyPlan(&planC2R_);
 135 }
 136
 137 void GpuParallel3dFft::perform3dFft(gmx_fft_direction  dir,
 138                                     CommandEvent      *timingEvent)
 139 {
 140     cl_mem                            tempBuffer = nullptr;
 141     constexpr std::array<cl_event, 0> waitEvents {{}};
 142
 143     clfftPlanHandle                   plan;
 144     clfftDirection                    direction;
 145     cl_mem *inputGrids, *outputGrids;
 146
 147     switch (dir)
 148     {
 149         case GMX_FFT_REAL_TO_COMPLEX:
 150             plan        = planR2C_;
 151             direction   = CLFFT_FORWARD;
 152             inputGrids  = &realGrid_;
 153             outputGrids = &complexGrid_;
 154             break;
 155         case GMX_FFT_COMPLEX_TO_REAL:
 156             plan        = planC2R_;
 157             direction   = CLFFT_BACKWARD;
 158             inputGrids  = &complexGrid_;
 159             outputGrids = &realGrid_;
 160             break;
 161         default:
 162             GMX_THROW(gmx::NotImplementedError("The chosen 3D-FFT case is not implemented on GPUs"));
 163     }
 164     handleClfftError(clfftEnqueueTransform(plan, direction,
 165                                            commandStreams_.size(), commandStreams_.data(),
 166                                            waitEvents.size(), waitEvents.data(), timingEvent,
 167                                            inputGrids, outputGrids, tempBuffer), "clFFT execution failure");
 168 }