From 57b5e5a4dba93c5720986c52a77549182776ae1a Mon Sep 17 00:00:00 2001 From: Erik Lindahl Date: Sun, 22 May 2016 18:40:43 +0200 Subject: [PATCH] Detect the usage of hwtop XML caching The environment variable HWLOC_XMLFILE can be used to speed up hwloc detection e.g. on Xeon Phi. Documented in the user guide, and made sure we detect when such caching was used to avoid strange future bug reports. Fixes #1946. Change-Id: Id99385fb7cc1e2692fb9b06fa187424058aaa213 --- docs/user-guide/environment-variables.rst | 7 +++++++ src/gromacs/hardware/detecthardware.cpp | 15 ++++++++++++++- src/gromacs/hardware/hardwaretopology.cpp | 8 ++++++-- src/gromacs/hardware/hardwaretopology.h | 12 ++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/environment-variables.rst b/docs/user-guide/environment-variables.rst index 01a62e5593..3c95d2663a 100644 --- a/docs/user-guide/environment-variables.rst +++ b/docs/user-guide/environment-variables.rst @@ -332,6 +332,13 @@ Performance and Run Control resolution of buffer size in Verlet cutoff scheme. The default value is 0.001, but can be overridden with this environment variable. +``HWLOC_XMLFILE`` + Not strictly a |Gromacs| environment variable, but on large machines + the hwloc detection can take a few seconds if you have lots of MPI processes. + If you run the hwloc command `lstopo out.xml` and set this environment + variable to point to the location of this file, the hwloc library will use + the cached information instead, which can be faster. + ``MPIRUN`` the ``mpirun`` command used by :ref:`gmx tune_pme`. diff --git a/src/gromacs/hardware/detecthardware.cpp b/src/gromacs/hardware/detecthardware.cpp index 30537e8765..6e351e6665 100644 --- a/src/gromacs/hardware/detecthardware.cpp +++ b/src/gromacs/hardware/detecthardware.cpp @@ -884,7 +884,11 @@ gmx_hw_info_t *gmx_detect_hardware(FILE *fplog, const t_commrec *cr, // TODO: Get rid of this altogether. hwinfo_g->nthreads_hw_avail = hwinfo_g->hardwareTopology->machine().logicalProcessorCount; - check_nthreads_hw_avail(cr, fplog, hwinfo_g->nthreads_hw_avail); + // If we detected the topology on this system, double-check that it makes sense + if (hwinfo_g->hardwareTopology->isThisSystem()) + { + check_nthreads_hw_avail(cr, fplog, hwinfo_g->nthreads_hw_avail); + } /* detect GPUs */ hwinfo_g->gpu_info.n_dev = 0; @@ -1054,6 +1058,15 @@ static std::string detected_hardware_string(const gmx_hw_info_t *hwinfo, break; } + if (!hwTop.isThisSystem()) + { + s += gmx::formatString(" NOTE: Hardware topology cached or synthetic, not detected.\n"); + if (char *p = getenv("HWLOC_XMLFILE")) + { + s += gmx::formatString(" HWLOC_XMLFILE=%s\n", p); + } + } + if (bFullCpuInfo) { if (hwTop.supportLevel() >= gmx::HardwareTopology::SupportLevel::Basic) diff --git a/src/gromacs/hardware/hardwaretopology.cpp b/src/gromacs/hardware/hardwaretopology.cpp index ac7750138c..38ff314c42 100644 --- a/src/gromacs/hardware/hardwaretopology.cpp +++ b/src/gromacs/hardware/hardwaretopology.cpp @@ -485,7 +485,8 @@ parseHwLocDevices(const hwloc_topology_t topo, void parseHwLoc(HardwareTopology::Machine * machine, - HardwareTopology::SupportLevel * supportLevel) + HardwareTopology::SupportLevel * supportLevel, + bool * isThisSystem) { hwloc_topology_t topo; @@ -505,7 +506,9 @@ parseHwLoc(HardwareTopology::Machine * machine, hwloc_topology_destroy(topo); return; // SupportLevel::None. } + // If we get here, we can get a valid root object for the topology + *isThisSystem = hwloc_topology_is_thissystem(topo); // Parse basic information about sockets, cores, and hardware threads if (parseHwLocSocketsCoresThreads(topo, machine) == 0) @@ -598,9 +601,10 @@ HardwareTopology HardwareTopology::detect() result.machine_.numa.baseLatency = 0.0; result.machine_.numa.maxRelativeLatency = 0.0; result.supportLevel_ = SupportLevel::None; + result.isThisSystem_ = true; #if GMX_HWLOC - parseHwLoc(&result.machine_, &result.supportLevel_); + parseHwLoc(&result.machine_, &result.supportLevel_, &result.isThisSystem_); #endif // If something went wrong in hwloc (or if it was not present) we might diff --git a/src/gromacs/hardware/hardwaretopology.h b/src/gromacs/hardware/hardwaretopology.h index 590f8a7f38..267e3eb3c3 100644 --- a/src/gromacs/hardware/hardwaretopology.h +++ b/src/gromacs/hardware/hardwaretopology.h @@ -207,6 +207,17 @@ class HardwareTopology SupportLevel supportLevel() const { return supportLevel_; } + /*! \brief Return true if we actually detected hardware. + * + * \return This method will normally return true, when we actually ran + * the hardware detection as part of this process to construct + * the object. It will be false when the object was constructed + * by reading a cached XML file, or possibly generated from + * synthetic data. + */ + bool + isThisSystem() const { return isThisSystem_; } + /*! \brief Return the machine topology tree * * You can always call this routine, but be aware that some or all contents @@ -241,6 +252,7 @@ class HardwareTopology SupportLevel supportLevel_; //!< Available topology information Machine machine_; //!< The machine map + bool isThisSystem_; //!< Machine map is real (vs. cached/synthetic) }; } -- 2.11.4.GIT