From 99fd3d66909d8cfae17329a70296ea14e3a9b5b6 Mon Sep 17 00:00:00 2001 From: Mark Abraham Date: Mon, 11 Mar 2013 13:30:59 +0100 Subject: [PATCH] Update management of linear algebra libraries Management of detection and/or linking to BLAS and LAPACK libraries is re-organized. The code has migrated to its own module. This will help future extension and maintenance. This version communicates things that are newsworthy and stays out of the way when nothing is changing. We no longer over-write the values specified by the user for GMX_EXTERNAL_(BLAS|LAPACK). Previously, this was used to signal whether detection succeeded, but that does not really get the job done. Instead, the user is notified that detection failed (repeatedly, if they deliberately set such an option on). Correct usage and expected behaviour in all cases is documented both in the code and the install guide. The user interface is pretty much unchanged. We still don't offer full configurability (e.g. MKL for FFTs must use MKL for linear algebra unless GMX_*_USER is used, and the only way to get MKL for linear algebra is to use it for FFTs). The size of any performance difference is probably very small, and if the user really needs mdrun with certain FFT and tools with certain linear algebra library, they can do two configurations. Note that mdrun never calls any linear algebra routines (tested empirically)! Expanded the solution of #771 by testing that the user supplied libraries that actually work. If not, we emit a warning and try to use them anyway. We also now check that MKL really does provide linear algebra routines, and fall back to the default treatment if it does not. Refs #771,#1186 Change-Id: Ife5c59694e29a3ce73fc55975e26f6c083317d9b --- CMakeLists.txt | 41 +------ admin/installguide/installguide.tex | 20 ++-- cmake/gmxManageLinearAlgebraLibraries.cmake | 167 ++++++++++++++++++++++++++++ src/gmxlib/CMakeLists.txt | 6 +- 4 files changed, 183 insertions(+), 51 deletions(-) create mode 100644 cmake/gmxManageLinearAlgebraLibraries.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 380b169c5d..4b4d6cd6d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -587,11 +587,6 @@ if(GMX_GPU) gmx_gpu_setup() endif(GMX_GPU) -if(APPLE) - find_library(ACCELERATE_FRAMEWORK Accelerate) - list(APPEND GMX_EXTRA_LIBRARIES ${ACCELERATE_FRAMEWORK}) -endif(APPLE) - if(CYGWIN) set(GMX_CYGWIN 1) endif(CYGWIN) @@ -1044,41 +1039,7 @@ if(FFTW3_THREADS OR FFTW3F_THREADS) add_definitions(-DFFT5D_FFTW_THREADS) endif() -set(GMX_EXTERNAL_BLAS TRUE CACHE BOOL "Use external BLAS instead of built-in") -set(GMX_EXTERNAL_LAPACK TRUE CACHE BOOL "Use external LAPACK instead of built-in") -# MKL has BLAS/LAPACK routines -if(NOT HAVE_LIBMKL AND NOT ACCELERATE_FRAMEWORK) - if(GMX_EXTERNAL_BLAS) - if (GMX_BLAS_USER) - list(APPEND GMX_EXTRA_LIBRARIES ${GMX_BLAS_USER}) - else(GMX_BLAS_USER) - set(BLAS_FIND_QUIETLY ON) - find_package(BLAS) - if (BLAS_FOUND) - list(APPEND GMX_EXTRA_LIBRARIES ${BLAS_LIBRARIES}) - else() - MESSAGE(STATUS "Using internal BLAS library") - set(GMX_EXTERNAL_BLAS FALSE CACHE BOOL "Use external BLAS instead of built-in" FORCE) - endif() - endif(GMX_BLAS_USER) - endif(GMX_EXTERNAL_BLAS) - if(GMX_EXTERNAL_LAPACK) - if (GMX_LAPACK_USER) - list(APPEND GMX_EXTRA_LIBRARIES ${GMX_LAPACK_USER}) - else(GMX_LAPACK_USER) - set(LAPACK_FIND_QUIETLY ON) - find_package(LAPACK) - if (LAPACK_FOUND) - list(APPEND GMX_EXTRA_LIBRARIES ${LAPACK_LIBRARIES}) - else() - MESSAGE(STATUS "Using internal LAPACK library") - set(GMX_EXTERNAL_LAPACK FALSE CACHE BOOL "Use external LAPACK instead of built-in" FORCE) - endif() - endif(GMX_LAPACK_USER) - endif(GMX_EXTERNAL_LAPACK) -endif() -mark_as_advanced(GMX_EXTERNAL_LAPACK) -mark_as_advanced(GMX_EXTERNAL_BLAS) +include(gmxManageLinearAlgebraLibraries) set(GMX_USE_PLUGINS OFF CACHE INTERNAL "Whether GROMACS will really try to compile support for VMD plugins") diff --git a/admin/installguide/installguide.tex b/admin/installguide/installguide.tex index aabe66a079..b5ede9a4d8 100644 --- a/admin/installguide/installguide.tex +++ b/admin/installguide/installguide.tex @@ -438,18 +438,22 @@ As mentioned above, sometimes vendor \blas{} and \lapack{} libraries can provide performance enhancements for \gromacs{} when doing normal-mode analysis or covariance analysis. For simplicity, the text below will refer only to \blas{}, but the same options are available -for \lapack{}. By default, the \cmake{} optionv -\verb+GMX_EXTERNAL_BLAS+ is on, which triggers \cmake{} to search for -\blas{}. If one is found, then it is used. Otherwise, \cmake{} falls -back on internal versions provided in the \gromacs{} source. These are -fine for normal use. If you need to specify a non-standard path to -search, use \verb+-DCMAKE_PREFIX_PATH=/path/to/search+. +for \lapack{}. By default, CMake will search for \blas{}, use it if it +is found, and otherwise fall back on a version of \blas{} internal to +\gromacs{}. The \cmake{} option \verb+GMX_EXTERNAL_BLAS+ will be set +accordingly. The internal versions are fine for normal use. If you +need to specify a non-standard path to search, use +\verb+-DCMAKE_PREFIX_PATH=/path/to/search+. If you need to specify a +library with a non-standard name (e.g. ESSL on AIX), then set +\verb+-DGMX_BLAS_USER=/path/to/reach/lib/libwhatever.a+. If you are using Intel's \mkl{} for \fft{}, then the \blas{} and -\lapack{} it provides are used automatically. +\lapack{} it provides are used automatically. This could be +over-ridden with \verb+GMX_BLAS_USER+, etc. On Apple platforms where the Accelerate Framework is available, these -will be automatically used for \blas{} and \lapack{}. +will be automatically used for \blas{} and \lapack{}. This could be +over-ridden with \verb+GMX_BLAS_USER+, etc. \subsection{Native GPU acceleration} If you have the \cuda{} Software Development Kit installed, you can diff --git a/cmake/gmxManageLinearAlgebraLibraries.cmake b/cmake/gmxManageLinearAlgebraLibraries.cmake new file mode 100644 index 0000000000..1392c0ad39 --- /dev/null +++ b/cmake/gmxManageLinearAlgebraLibraries.cmake @@ -0,0 +1,167 @@ +# Helper macro for the function below. We treat BLAS and LAPACK the same +# way, so there's no reason to duplicate the logic for them +# +# MESSSAGE_TEXT variable is accumulated while checks for these +# libraries fail, but the message is only emitted if we are forced to +# fall back on the internal version. +# +# Arguments should be: +# name "BLAS" or "LAPACK" +# function_in_library the name of a function to use in a linking test of that library +macro(manage_linear_algebra_library name function_in_library) + set(_library_was_found 0) + set(_find_quietly FALSE) + set(_user_var_changed FALSE) + if(NOT DEFINED GMX_EXTERNAL_${name} OR + (GMX_EXTERNAL_${name} AND NOT "${GMX_${name}_USER}" STREQUAL "${GMX_${name}_USER_PREV}")) + set(_user_var_changed TRUE) + endif() + if(DEFINED GMX_EXTERNAL_${name} AND NOT _user_var_changed) + set(_find_quietly TRUE) + endif() + + # We could consider printing status messages at the beginning and + # end, which would require caching whether the previous provider + # was user/MKL/external/internal. It's possible (we do it for + # FFT), but the number of times the user changes these is pretty + # low, so let's solve that one in master branch when we have + # better CMake gear to support it. + if(GMX_EXTERNAL_${name} OR NOT DEFINED GMX_EXTERNAL_${name}) + set(GMX_${name}_USER_PREV ${GMX_${name}_USER} CACHE INTERNAL + "Previous value of GMX_${name}_USER (to detect changes)" FORCE) + set(_message_text) + # Check for user-specified libraries if external libraries have + # been specified (which is the default). + if(GMX_${name}_USER) + set(_libraries_to_link ${GMX_${name}_USER}) + set(_library_was_found 1) + + if(NOT _find_quietly) + set(CMAKE_REQUIRED_LIBRARIES ${GMX_${name}_USER}) + if(_user_var_changed) + unset(_${name}_user_works CACHE) + endif() + message(STATUS "Checking that user ${name} library ${GMX_${name}_USER} works") + check_function_exists(${function_in_library} _${name}_user_works) + if(NOT _${name}_user_works) + message(WARNING "GMX_${name}_USER library ${GMX_${name}_USER} was specified, but it may not provide ${name}. We are proceeding by assuming you know what you are doing and that linking F77-style to this library will work.") + endif() + + if(HAVE_LIBMKL) + message(STATUS "MKL and GMX_${name}_USER were both specified. Using the latter for ${name}.") + endif() + endif() + endif() + + if(NOT _library_was_found AND HAVE_LIBMKL) + set(CMAKE_REQUIRED_LIBRARIES ${FFT_LINKER_FLAGS} ${FFT_LIBRARIES}) + # This may also not work correctly if the user changes + # MKL_LIBRARIES after the first run. However, + # MKL_LIBRARIES is only needed for icc version < 11, or + # for trying to use MKL with a non-Intel compiler, and we + # can live with that for now. + check_function_exists(${function_in_library} _${name}_mkl_works) + if(_${name}_mkl_works) + # If we ever need to compile with MKL linear algebra and + # not with FFT supplied by MKL, uncomment the next line + # (and probably tweak other things). +# list(APPEND LINEAR_ALGEBRA_LIBRARIES ${FFT_LINKER_FLAGS} ${FFT_LIBRARIES}) + set(_library_was_found 1) + else() + set(_message_text "Intel's MKL was specified, and it should provide ${name}, but it does not. ") + endif() + endif() + + # If detection of ${name} has never run, or none of the preceding + # detection succeeded, try to detect ${name} in the CMake + # detection paths, etc. + if (NOT _library_was_found) + set(${name}_FIND_QUIETLY _find_quietly) + # Note that this finds all kinds of system libraries, + # including Apple's Accelerate Framework (and perhaps MKL for + # icc < 11). + find_package(${name}) + if (${name}_FOUND) + set(_libraries_to_link ${${name}_LIBRARIES}) + set(_library_was_found 1) + endif() + endif() + + if (NOT _library_was_found AND NOT _find_quietly) + message("${_message_text}A ${name} library was not found by CMake in the paths available to it. Falling back on the GROMACS internal version of the ${name} library instead. This is fine for normal usage.") + endif() + endif() + + # Default behaviour is to try to use an external library, but fall + # back on the internal one if none is found. + set(GMX_EXTERNAL_${name} ${_library_was_found} CACHE BOOL "Use a ${name} library that is external to GROMACS if possible (ON), or the internal GROMACS one (OFF)") + mark_as_advanced(GMX_EXTERNAL_${name}) + # Default behaviour is to use a library found on the system or in + # GROMACS. The user must actively set GMX_${name}_USER if they + # want to specify a library. + set(GMX_${name}_USER "" CACHE BOOL "Use a ${name} library found on the system (OFF), or a ${name} library supplied by the user (any other value, which is a full path to that ${name} library)") + mark_as_advanced(GMX_${name}_USER) + + if(GMX_EXTERNAL_${name}) + if (NOT _library_was_found) + message(FATAL_ERROR "You have set GMX_EXTERNAL_${name}=ON to instruct GROMACS to use an external ${name} library, but no external library could be detected.") + endif () + # Actually trigger linking. + list(APPEND LINEAR_ALGEBRA_LIBRARIES ${_libraries_to_link}) + else() + # Triggering the compilation of the internal version of the library is handled elsewhere. + endif() +endmacro() + +# The default behaviour is to try to detect an "external" BLAS and/or +# LAPACK, perhaps provided by a vendor, use those if found, and +# otherwise fall back on the GROMACS internal implementations of +# these. If the libraries are not in a standard location, the user can +# indicate a search path with CMAKE_PREFIX_PATH. +# +# However, if we are using icc+mkl (so a build command that includes +# -mkl), then it is probably painful to try to link some other BLAS or +# LAPACK. In that case, we use the BLAS & LAPACK provided by MKL. In +# principle, we could offer a more configurable behaviour if/when +# there is need to (say) use vendor BLAS with MKL for FFTs. +# +# If the vendor BLAS and/or LAPACK have abnormal library names, then +# the default searching procedure will fail (e.g. Redmine #771). The +# GMX_(BLAS|LAPACK)_USER variables can be used to indicate the correct +# libraries. If these do not work, a warning is emitted and we try to +# use them anyway, assuming the user knows what they are doing. + +# Inputs: +# GMX_EXTERNAL_BLAS user input about whether to detect BLAS +# GMX_EXTERNAL_LAPACK user input about whether to detect LAPACK +# HAVE_LIBMKL true if the build will link to MKL +# FFT_LINKER_FLAGS used iff HAVE_MKL +# FFT_LIBRARIES used iff HAVE_MKL +# GMX_BLAS_USER user input for BLAS libraries to use +# GMX_LAPACK_USER user input for LAPACK libraries to use +# +# This function sets the following cache variables: +# GMX_EXTERNAL_BLAS according to whether external BLAS is being used +# GMX_EXTERNAL_LAPACK according to whether external LAPACK is being used +# GMX_BLAS_USER off = use a system library; +# any other value = full path to the library to use +# GMX_LAPACK_USER off = use a system library; +# any other value = full path to the library to use +# +# This function sets the following variables in its parent scope: +# LINEAR_ALGEBRA_LIBRARIES will be set as required to add libraries required for linear algebra +# +function(gmxManageLinearAlgebraLibraries) + include(CheckFunctionExists) + # Probably not necessary to unset, but let's be clear about usage. + unset(LINEAR_ALGEBRA_LIBRARIES) + + manage_linear_algebra_library(BLAS dgemm_) + set(BLAS_FIND_QUIETLY ON) + manage_linear_algebra_library(LAPACK cheev_) + + # Propagate the new local value to the parent scope + set(LINEAR_ALGEBRA_LIBRARIES "${LINEAR_ALGEBRA_LIBRARIES}" PARENT_SCOPE) +endfunction() + +gmxManageLinearAlgebraLibraries() diff --git a/src/gmxlib/CMakeLists.txt b/src/gmxlib/CMakeLists.txt index 735ce020ff..debfceb691 100644 --- a/src/gmxlib/CMakeLists.txt +++ b/src/gmxlib/CMakeLists.txt @@ -77,11 +77,11 @@ endif() if(NOT GMX_EXTERNAL_BLAS) file(GLOB BLAS_SOURCES gmx_blas/*.c) -endif(NOT GMX_EXTERNAL_BLAS) +endif() if(NOT GMX_EXTERNAL_LAPACK) file(GLOB LAPACK_SOURCES gmx_lapack/*.c) -endif(NOT GMX_EXTERNAL_LAPACK) +endif() # This would be the standard way to include thread_mpi, but we want libgmx # to link the functions directly @@ -113,7 +113,7 @@ endif() # NONBONDED_SOURCES is imported from the nonbonded subdirectory. add_library(gmx ${GMXLIB_SOURCES} ${BLAS_SOURCES} ${LAPACK_SOURCES} ${THREAD_MPI_SRC} ${NONBONDED_SOURCES}) -target_link_libraries(gmx ${FFT_LIBRARIES} ${GMX_GPU_LIBRARIES} ${GMX_EXTRA_LIBRARIES} ${THREAD_LIB} ${OpenMP_SHARED_LINKER_FLAGS}) +target_link_libraries(gmx ${FFT_LIBRARIES} ${LINEAR_ALGEBRA_LIBRARIES} ${GMX_GPU_LIBRARIES} ${GMX_EXTRA_LIBRARIES} ${THREAD_LIB} ${OpenMP_SHARED_LINKER_FLAGS}) if(USE_VERSION_H) add_dependencies(gmx gmx_version) endif() -- 2.11.4.GIT