Merge pull request #320 from ye-luo/fix-restart-final

Fix restart IO

CMakeLists.txt

@@ -231,7 +231,7 @@ SET(OHMMS_ORTHO 0)
 #list of options to determine libraries. First, try to use libraries available
 #FindXYZ.cmake will set the values if successful
 ######################################################################
-SET(ENABLE_PHDF5 0 CACHE BOOL "Enable phdf5 for output")
+SET(ENABLE_PHDF5 1 CACHE BOOL "Enable phdf5 for output")
 SET(HAVE_LIBXML2 1)
 SET(HAVE_LIBFFTW 0)
 SET(HAVE_LIBXMLPP 0)
@@ -654,6 +654,20 @@ IF(HDF5_FOUND)
   #IF(SZLIB_FOUND)
   #  SET(QMC_UTIL_LIBS ${QMC_UTIL_LIBS} ${SZLIB_LIBRARIES})
   #ENDIF(SZLIB_FOUND)
+  IF(HDF5_IS_PARALLEL)
+    IF(ENABLE_PHDF5)
+      MESSAGE(STATUS "Using HDF5 parallel collective I/O")
+    ELSE(ENABLE_PHDF5)
+      MESSAGE(STATUS "Using HDF5 non-scalable serial I/O")
+    ENDIF(ENABLE_PHDF5)
+  ELSE(HDF5_IS_PARALLEL)
+    IF(ENABLE_PHDF5)
+      MESSAGE(STATUS "Using HDF5 non-scalable serial I/O due to the lack of library support for parallel")
+      SET(ENABLE_PHDF5 0)
+    ELSE(ENABLE_PHDF5)
+      MESSAGE(STATUS "Using HDF5 non-scalable serial I/O")
+    ENDIF(ENABLE_PHDF5)
+  ENDIF(HDF5_IS_PARALLEL)
 ELSE(HDF5_FOUND)
   if(NOT QMC_PHI)
     MESSAGE(FATAL_ERROR "Require hdf5 1.6.4 or higher. Set HDF5_ROOT")

config/BGQ_Clang++11_ToolChain.cmake

@@ -28,7 +28,7 @@ SET(BOOST_ROOT /soft/libraries/boost/1.62.0/cnk-bgclang++11/current/)
 
 SET(CMAKE_FIND_ROOT_PATH
      /home/projects/qmcpack/libXML2-2.9.1
-     /soft/libraries/hdf5/1.8.14/cnk-gcc/current
+     /soft/libraries/hdf5/current/cnk-gcc/current
      /soft/libraries/alcf/current/gcc/FFTW3
      /soft/libraries/alcf/current/gcc/ZLIB
 )

src/OhmmsApp/RandomNumberControl.cpp

@@ -12,8 +12,6 @@
 //
 // File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
 //////////////////////////////////////////////////////////////////////////////////////
-    
-    
 
 
 #include <Configuration.h>
@@ -239,7 +237,7 @@ bool RandomNumberControl::put(xmlNodePtr cur)
   return true;
 }
 
-void RandomNumberControl::read(const std::string& fname, Communicate* comm)
+void RandomNumberControl::read_old(const std::string& fname, Communicate* comm)
 {
   int nthreads=omp_get_max_threads();
   std::vector<uint_type> vt_tot, vt;
@@ -321,7 +319,7 @@ void RandomNumberControl::read(const std::string& fname, Communicate* comm)
   }
 }
 
-void RandomNumberControl::write(const std::string& fname, Communicate* comm)
+void RandomNumberControl::write_old(const std::string& fname, Communicate* comm)
 {
   int nthreads=omp_get_max_threads();
   std::vector<uint_type> vt, vt_tot;
@@ -375,4 +373,238 @@ void RandomNumberControl::write(const std::string& fname, Communicate* comm)
 #endif
   }
 }
+
+/*New functions past this point*/
+//switch between read functions
+void RandomNumberControl::read(const std::string& fname, Communicate* comm)
+{
+  std::string h5name=fname+".random.h5";
+  hdf_archive hin(comm, true); //attempt to read in parallel
+  hin.open(h5name,H5F_ACC_RDONLY);
+  if(hin.is_parallel())
+    read_parallel(hin, comm);
+  else
+    read_rank_0(hin, comm);
+}
+
+//switch between write functions
+void RandomNumberControl::write(const std::string& fname, Communicate* comm)
+{
+  std::string h5name=fname+".random.h5";
+  hdf_archive hout(comm, true); //attempt to write in parallel
+  hout.create(h5name);
+  if(hout.is_parallel())
+    write_parallel(hout, comm);
+  else
+    write_rank_0(hout, comm);
+}
+
+//Parallel read
+void RandomNumberControl::read_parallel(hdf_archive& hin, Communicate* comm)
+{
+  int nthreads = omp_get_max_threads();
+  std::vector<uint_type> vt, mt;
+  TinyVector<int,3> shape_now(comm->size(), nthreads, Random.state_size()); //cur configuration
+  TinyVector<int,3> shape_hdf5(3,0); //configuration when file was written
+
+  //grab shape and Random.state_size() used to create hdf5 file
+  hin.push(hdf::main_state);
+  hin.read(shape_hdf5, "nprocs_nthreads_statesize");
+
+  //if hdf5 file's shape and the current shape don't match, abort read
+  if(shape_hdf5[0] != shape_now[0] || shape_hdf5[1] != shape_now[1] || shape_hdf5[2] != shape_now[2])
+  {
+    app_log() << "Mismatched random number generators."
+      << "\n  Number of procs in streams : old=" << shape_hdf5[0] << " new= " << shape_now[0]
+      << "\n  Number of threads in streams : old=" << shape_hdf5[1] << " new= " << shape_now[1]
+      << "\n  State size per stream : old=" << shape_hdf5[2] << " new= " << shape_now[2]
+      << "\n  Using the random streams generated at the initialization.\n";
+    return;
+  }
+  app_log() << "  Restart from the random number streams from the previous configuration.\n";
+
+  TinyVector<int,2> shape(comm->size()*nthreads, Random.state_size()); //global dims of children dataset
+  vt.resize(nthreads*Random.state_size()); //buffer for children[ip]
+  mt.resize(Random.state_size()); //buffer for single thread Random object of random nums
+
+  TinyVector<int,2> counts(nthreads, Random.state_size()); //local dimensions of dataset
+  TinyVector<int,2> offsets(comm->rank() * nthreads, 0); //offsets for each process to read in
+
+  hin.push("random"); //group that holds children[ip] random nums
+  hyperslab_proxy<std::vector<uint_type>,2> slab(vt, shape, counts, offsets);
+  hin.read(slab,Random.EngineName);
+
+  hin.pop();
+  hin.push("random_master"); //group that holds Random_th random nums
+  shape[0] = comm->size(); //reset shape, counts and offset for non-multiple threads
+  counts[0] = 1;
+  offsets[0] = comm->rank();
+  hyperslab_proxy<std::vector<uint_type>,2> slab2(mt, shape, counts, offsets);
+  hin.read(slab2,Random.EngineName);
+  hin.close();
+
+  std::vector<uint_type>::iterator vt_it(vt.begin());
+  for(int ip=0; ip<nthreads; ip++, vt_it += shape[1])
+  {
+    std::vector<uint_type> c(vt_it,vt_it+shape[1]);
+    Children[ip]->load(c); //load random nums back to program from buffer
+  }
+  Random.load(mt); //load random nums back to prog from buffer
+}
+
+//Parallel write
+void RandomNumberControl::write_parallel(hdf_archive& hout, Communicate* comm)
+{
+  int nthreads=omp_get_max_threads();
+  std::vector<uint_type> vt, mt;
+  TinyVector<int,3> shape_hdf5(comm->size(), nthreads, Random.state_size()); //configuration at write time
+  vt.reserve(nthreads*Random.state_size()); //buffer for random numbers from children[ip] of each thread
+  mt.reserve(Random.state_size()); //buffer for random numbers from single Random object
+
+  for(int ip=0; ip<nthreads; ++ip)
+  {
+    std::vector<uint_type> c;
+    Children[ip]->save(c);
+    vt.insert(vt.end(),c.begin(),c.end()); //get nums from each thread into buffer
+  }
+  Random.save(mt); //get nums for single random object (no threads)
+
+  TinyVector<int,2> shape(comm->size()*nthreads,Random.state_size()); //global dimensions
+  TinyVector<int,2> counts(nthreads, Random.state_size()); //local dimensions
+  TinyVector<int,2> offsets(comm->rank() * nthreads, 0); //offset for the file write
+
+  hout.push(hdf::main_state);
+  hout.write(shape_hdf5, "nprocs_nthreads_statesize"); //save the shape of the data at write
+
+  hout.push("random"); //group for children[ip]
+  hyperslab_proxy<std::vector<uint_type>,2> slab(vt, shape, counts, offsets);
+  hout.write(slab,Random.EngineName); //write to hdf5file
+  hout.pop();
+
+  shape[0] = comm->size(); //adjust shape, counts, offset for just one thread
+  counts[0] = 1;
+  offsets[0] = comm->rank();
+  hout.push("random_master"); //group for random object without threads
+  hyperslab_proxy<std::vector<uint_type>,2> slab2(mt, shape, counts, offsets);
+  hout.write(slab2,Random.EngineName); //write data to hdf5 file
+  hout.close();
+}
+
+//Scatter read
+void RandomNumberControl::read_rank_0(hdf_archive& hin, Communicate* comm)
+{
+  int nthreads = omp_get_max_threads();
+  std::vector<uint_type> vt, vt_tot, mt, mt_tot;
+  TinyVector<int,3> shape_now(comm->size(), nthreads, Random.state_size()); //current configuration
+  TinyVector<int,2> shape(comm->size()*nthreads, Random.state_size()); //dimensions of children dataset
+  TinyVector<int,3> shape_hdf5(3,0); //configuration when hdf5 file was written
+
+  //grab configuration of threads/procs and Random.state_size() in hdf5 file
+  if(comm->rank() == 0)
+  {
+    hin.push(hdf::main_state);
+    hin.read(shape_hdf5, "nprocs_nthreads_statesize");
+  }
+
+  mpi::bcast(*comm, shape_hdf5);
+
+  //if hdf5 file's configuration and current configuration don't match, abort read
+  if(shape_hdf5[0] != shape_now[0] || shape_hdf5[1] != shape_now[1] || shape_hdf5[2] != shape_now[2])
+  {
+    app_log() << "Mismatched random number generators."
+      << "\n  Number of procs in streams : old=" << shape_hdf5[0] << " new= " << shape_now[0]
+      << "\n  Number of threads in streams : old=" << shape_hdf5[1] << " new= " << shape_now[1]
+      << "\n  State size per stream : old=" << shape_hdf5[2] << " new= " << shape_now[2]
+      << "\n  Using the random streams generated at the initialization.\n";
+    return;
+  }
+  app_log() << "  Restart from the random number streams from the previous configuration.\n";
+
+  vt.resize(nthreads*Random.state_size()); //buffer for random nums in children of each thread
+  mt.resize(Random.state_size()); //buffer for random numbers from single Random object
+
+  if(comm->rank() == 0)
+  {
+    hin.push("random"); //group for children[ip] (Random.object for each thread)
+    vt_tot.resize(nthreads*Random.state_size()*comm->size());
+    hyperslab_proxy<std::vector<uint_type>,2> slab(vt_tot, shape);
+    hin.read(slab,Random.EngineName);
+    hin.pop();
+
+    shape[0] = comm->size(); //reset shape to one thread per process
+    mt_tot.resize(Random.state_size()*comm->size());
+    hin.push("random_master"); //group for single Random object
+    hyperslab_proxy<std::vector<uint_type>,2> slab2(mt_tot, shape);
+    hin.read(slab2,Random.EngineName);
+    hin.close();
+  }
+
+  if(comm->size()>1)
+  {
+    mpi::scatter(*comm,vt_tot,vt); //divide big buffer into on for each proc
+    mpi::scatter(*comm,mt_tot,mt);
+  }
+  else
+  {
+    copy(vt_tot.begin(),vt_tot.end(),vt.begin());
+    copy(mt_tot.begin(),mt_tot.end(),mt.begin());
+  }
+
+  std::vector<uint_type>::iterator vt_it(vt.begin());
+  for(int i=0; i<nthreads; i++, vt_it += shape[1])
+  {
+    std::vector<uint_type> c(vt_it,vt_it+shape[1]);
+    Children[i]->load(c); //read seeds for each thread from buffer back into object
+  }
+  Random.load(mt); //read seeds back into object
+}
+
+//scatter write
+void RandomNumberControl::write_rank_0(hdf_archive& hout, Communicate* comm)
+{
+  int nthreads = omp_get_max_threads();
+  std::vector<uint_type> vt, vt_tot, mt, mt_tot;
+  TinyVector<int,2> shape(comm->size()*nthreads, Random.state_size()); //dimensions of children dataset
+  TinyVector<int,3> shape_hdf5(comm->size(), nthreads, Random.state_size()); //configuration at write time
+  vt.reserve(nthreads*Random.state_size()); //buffer for children[ip] (Random object of seeds for each thread)
+  mt.reserve(Random.state_size());  //buffer for single Random object of seeds, one per proc regardless of thread num
+
+  for(int i=0; i<nthreads; ++i)
+  {
+    std::vector<uint_type> c;
+    Children[i]->save(c);
+    vt.insert(vt.end(),c.begin(),c.end()); //copy children[nthreads] seeds to buffer
+  }
+  Random.save(mt); //copy random_th seeds to buffer
+
+  if(comm->size()>1)
+  {
+    vt_tot.resize(vt.size()*comm->size());
+    mt_tot.resize(mt.size()*comm->size());
+    mpi::gather(*comm,vt,vt_tot); //gather into one big buffer for master write
+    mpi::gather(*comm,mt,mt_tot);
+  }
+  else
+  {
+    vt_tot=vt;
+    mt_tot=mt;
+  }
+
+  if(comm->rank()==0)
+  {
+    hout.push(hdf::main_state);
+    hout.write(shape_hdf5, "nprocs_nthreads_statesize"); //configuration at write time to file
+
+    hout.push("random"); //group for children[ip]
+    hyperslab_proxy<std::vector<uint_type>, 2> slab(vt_tot, shape);
+    hout.write(slab, Random.EngineName);
+    hout.pop();
+
+    shape[0] = comm->size(); //reset dims for single thread use
+    hout.push("random_master"); //group for random_th object
+    hyperslab_proxy<std::vector<uint_type>,2> slab2(mt_tot, shape);
+    hout.write(slab2,Random.EngineName);
+    hout.close();
+  }
+}
 }

src/OhmmsApp/RandomNumberControl.h

@@ -10,8 +10,6 @@
 //
 // File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
 //////////////////////////////////////////////////////////////////////////////////////
-    
-    
 
 
 #ifndef OHMMS_RANDOMNUMBERCONTROL_H__
@@ -19,6 +17,7 @@
 #include "OhmmsData/OhmmsElementBase.h"
 #include "Utilities/RandomGenerator.h"
 #include "Utilities/PrimeNumberSet.h"
+#include <io/hdf_archive.h>
 
 class Communicate;
 
@@ -56,17 +55,48 @@ public:
 
   xmlNodePtr initialize(xmlXPathContextPtr);
 
-  /** read random state from a hdf file
+  /** read in parallel or serial
    * @param fname file name
-   * @param comm communicator so that everyone reads its own data
+   * @param comm communicator
    */
   static void read(const std::string& fname, Communicate* comm);
-  /** write random state to a hdf file
+  /** write in parallel or serial
    * @param fname file name
-   * @param comm communicator so that everyone writes its own data
+   * @param comm communicator
    */
   static void write(const std::string& fname, Communicate* comm);
-
+  /** read random state from a hdf file in parallel
+   * @param hdf_archive set to parallel
+   * @param comm communicator
+   */
+  static void read_parallel(hdf_archive& hin, Communicate* comm);
+  /** write random state to a hdf file in parallel
+   * @param hdf_archive set to parallel
+   * @param comm communicator
+   */
+  static void write_parallel(hdf_archive& hout, Communicate* comm);
+  /** rank 0 reads random states from a hdf file
+   * and distributes them to all the other ranks
+   * @param hdf_archive set to serial
+   * @param comm communicator
+   */
+  static void read_rank_0(hdf_archive& hin, Communicate* comm);
+  /** rank 0 gathers the random states from all the other ranks
+   * and write them to a hdf file
+   * @param hin hdf_archive object set to serial
+   * @param comm communicator
+   */
+  static void write_rank_0(hdf_archive& hout, Communicate* comm);
+   /** read random state from a xml file
+   * @param fname file name
+   * @param comm communicator
+   */
+  static void read_old(const std::string& fname, Communicate* comm);
+   /** write random state to a xml file
+   * @param fname file name
+   * @param comm communicator
+   */
+  static void write_old(const std::string& fname, Communicate* comm);
 private:
 
   bool NeverBeenInitialized;

src/Particle/HDFWalkerInput_0_4.cpp

@@ -126,7 +126,11 @@ bool HDFWalkerInput_0_4::put(xmlNodePtr cur)
     FileStack.pop();
     std::string h5name(FileName);
     //success |= read_hdf5_scatter(h5name);
+#ifdef ENABLE_PHDF5
+    success |= read_phdf5(h5name);
+#else
     success |= read_hdf5(h5name);
+#endif
   }
   return success;
 #endif
@@ -286,55 +290,68 @@ bool HDFWalkerInput_0_4::read_hdf5_scatter( std::string h5name)
 
 bool HDFWalkerInput_0_4::read_phdf5( std::string h5name)
 {
-//Broken
+
-//#if defined(H5_HAVE_PARALLEL) && defined(ENABLE_PHDF5)
+  int nw_in=0;
-//  hin.read(aversion,hdf::version);
+  h5name.append(hdf::config_ext);
-//  int found_group=hin.is_group(hdf::main_state);
+  hdf_archive hin(myComm,true); //everone reads this
-//  if(!found_group)
+  bool success=hin.open(h5name,H5F_ACC_RDONLY);
-//    continue;
+  //check if hdf and xml versions can work together
-//  //start main_state
+  HDFVersion aversion;
-//  hin.push(hdf::main_state);
+
-//  int nw_in=0;
+  hin.read(aversion,hdf::version);
-//  hin.read(nw_in,hdf::num_walkers);
+  if(!(aversion < i_info.version))
-//  std::vector<int> woffsets(myComm->size()+1,0);
+  {
-//  FairDivideLow(nw_in,myComm->size(),woffsets);
+    int found_group=hin.is_group(hdf::main_state);
-//  int mynode=myComm->rank();
+    hin.push(hdf::main_state);
-//  int nw_loc=woffsets[mynode+1]-woffsets[mynode];
+    hin.read(nw_in,hdf::num_walkers);
-//  std::cout << "node =" << mynode << " nw_loc=" << nw_loc << " nw_in=" << nw_in << std::endl;
+  }
-//  //TinyVector<hsize_t,3> gcounts, counts, offset;
+  else
-//  hsize_t gcounts[3], counts[3], offset[3];
+  {
-//  gcounts[0]=nw_in;
+    app_error() << " Mismatched version. xml = " << i_info.version << " hdf = " << aversion << std::endl;
-//  gcounts[1]=targetW.getTotalNum();
+  }
-//  gcounts[2]=OHMMS_DIM;
+
-//  counts[0]=nw_loc;
+  if(nw_in==0)
-//  counts[1]=targetW.getTotalNum();
+  {
-//  counts[2]=OHMMS_DIM;
+    app_error() << " No walkers in " << h5name << std::endl;
-//  offset[0]=woffsets[mynode];
+    return false;
-//  offset[1]=0;
+  }
-//  offset[2]=0;
+
-//  int nitems=targetW.getTotalNum()*OHMMS_DIM;
+  typedef std::vector<QMCTraits::RealType>  Buffer_t;
-//  typedef std::vector<QMCTraits::RealType>  Buffer_t;
+  Buffer_t posin;
-//  Buffer_t posin(nw_loc*nitems);
+  TinyVector<int,3> dims(nw_in,targetW.getTotalNum(), OHMMS_DIM);
-//  hid_t dataset = H5Dopen(hin.top(),hdf::walkers);
+
-//  hid_t dataspace = H5Dget_space(dataset);
+  std::vector<int> woffsets;
-//  int rank_n = H5Sget_simple_extent_ndims(dataspace);
+  hin.read(woffsets,"walker_partition");
-//  int status_n = H5Sget_simple_extent_dims(dataspace, gcounts, NULL);
+  int nw_loc=woffsets[myComm->rank()+1]-woffsets[myComm->rank()];
-//  hid_t type_id=get_h5_datatype(posin[0]);
+
-//  hid_t memspace = H5Screate_simple(3, counts, NULL);
+  int np1=myComm->size()+1;
-//  herr_t status = H5Sselect_hyperslab(dataspace,H5S_SELECT_SET, offset,NULL,counts,NULL);
+  if(woffsets.size()!= np1)
-//  status = H5Dread(dataset, type_id, memspace, dataspace, hin.xfer_plist, &posin[0]);
+  {
-//  H5Sclose(dataspace);
+    woffsets.resize(myComm->size()+1,0);
-//  H5Sclose(memspace);
+    FairDivideLow(nw_in,myComm->size(),woffsets);
-//  H5Dclose(dataset);
+    nw_loc=woffsets[myComm->rank()+1]-woffsets[myComm->rank()];
-//  int curWalker = targetW.getActiveWalkers();
+  }
-//  targetW.createWalkers(nw_loc);
+
-//  Buffer_t::iterator it(posin.begin());
+  TinyVector<int,3> counts(nw_loc, targetW.getTotalNum(), OHMMS_DIM);
-//  for(int i=0,iw=curWalker; i<nw_loc; ++i,++iw)
+  TinyVector<int,3> offsets(woffsets[myComm->rank()], 0, 0);
-//  {
+  posin.resize(nw_loc*dims[1]*dims[2]);
-//    copy(it,it+nitems,get_first_address(targetW[iw]->R));
+
-//    it += nitems;
+  hyperslab_proxy<Buffer_t,3> slab(posin, dims, counts, offsets);
-//  }
+  hin.read(slab,hdf::walkers);
-//#endif
+
+  app_log() << " HDFWalkerInput_0_4::put getting " << dims[0] << " walkers " << posin.size() << std::endl;
+  nw_in=woffsets[myComm->rank()+1]-woffsets[myComm->rank()];
+  {
+    int nitems=targetW.getTotalNum()*OHMMS_DIM;
+    int curWalker = targetW.getActiveWalkers();
+    targetW.createWalkers(nw_in);
+    Buffer_t::iterator it(posin.begin());
+    for(int i=0,iw=curWalker; i<nw_in; ++i,++iw)
+    {
+      copy(it,it+nitems,get_first_address(targetW[iw]->R));
+      it += nitems;
+    }
+  }
   return true;
 }
 bool HDFWalkerInput_0_4::read_adios(xmlNodePtr cur)

src/Particle/HDFWalkerOutput.cpp

@@ -202,15 +202,11 @@ void HDFWalkerOutput::write_configuration(MCWalkerConfiguration& W, hdf_archive&
   hout.write(number_of_walkers,hdf::num_walkers);
 
   TinyVector<int,3> gcounts(number_of_walkers,number_of_particles,OHMMS_DIM);
-  //vector<int> gcounts(3);
-  //gcounts[0]=number_of_walkers;
-  //gcounts[1]=number_of_particles;
-  //gcounts[2]=OHMMS_DIM;
 
-  if(hout.is_collective())
+  if(hout.is_parallel())
   { 
     TinyVector<int,3> counts(W.getActiveWalkers(),            number_of_particles,OHMMS_DIM);
-    TinyVector<int,3> offsets(W.WalkerOffsets[myComm->rank()],number_of_particles,OHMMS_DIM);
+    TinyVector<int,3> offsets(W.WalkerOffsets[myComm->rank()],0,0);
     hyperslab_proxy<BufferType,3> slab(*RemoteData[0],gcounts,counts,offsets);
     hout.write(slab,hdf::walkers);
   }

src/io/hdf_archive.cpp

@@ -16,17 +16,17 @@
 namespace qmcplusplus
 {
 //const hid_t hdf_archive::is_closed;
-hdf_archive::hdf_archive(Communicate* c, bool use_collective)
+hdf_archive::hdf_archive(Communicate* c, bool request_pio)
   : file_id(is_closed), access_id(H5P_DEFAULT), xfer_plist(H5P_DEFAULT)
 {
   H5Eget_auto (&err_func, &client_data);
   H5Eset_auto (NULL, NULL);
-  set_access_plist(use_collective,c);
+  set_access_plist(request_pio,c);
 }
 
 hdf_archive::~hdf_archive()
 {
-#if defined(H5_HAVE_PARALLEL) && defined(ENABLE_PHDF5)
+#if defined(ENABLE_PHDF5)
   if(xfer_plist != H5P_DEFAULT) H5Pclose(xfer_plist);
   if(access_id != H5P_DEFAULT) H5Pclose(access_id);
 #endif
@@ -48,42 +48,50 @@ void hdf_archive::close()
   file_id=is_closed;
 }
 
-void hdf_archive::set_access_plist(bool use_collective, Communicate* comm)
+void hdf_archive::set_access_plist(bool request_pio, Communicate* comm)
 {
   access_id=H5P_DEFAULT;
   if(comm && comm->size()>1) //for parallel communicator
   {
-    bool use_pdf=false;
+    bool use_phdf5=false;
-    if(use_collective)
+    if(request_pio)
     {
-#if defined(H5_HAVE_PARALLEL) && defined(ENABLE_PHDF5)
+#if defined(ENABLE_PHDF5)
-//      MPI_Info info=MPI_INFO_NULL;
+      // enable parallel I/O
-//      access_id = H5Pcreate(H5P_FILE_ACCESS);
+      MPI_Info info=MPI_INFO_NULL;
-//      hid_t ret=H5Pset_fapl_mpio(access_id,comm->getMPI(),info);
+      access_id = H5Pcreate(H5P_FILE_ACCESS);
-//      xfer_plist = H5Pcreate(H5P_DATASET_XFER);
+#if H5_VERSION_GE(1,10,0)
-//      H5Pset_dxpl_mpio(xfer_plist,H5FD_MPIO_COLLECTIVE);
+      H5Pset_all_coll_metadata_ops(access_id,true);
-//      use_pdf=true;
+      H5Pset_coll_metadata_write(access_id,true);
-//      use_collective=false; // everynode writes something
+#endif
+      H5Pset_fapl_mpio(access_id,comm->getMPI(),info);
+      xfer_plist = H5Pcreate(H5P_DATASET_XFER);
+      // enable parallel collective I/O
+      H5Pset_dxpl_mpio(xfer_plist,H5FD_MPIO_COLLECTIVE);
+      use_phdf5=true;
+#else
+      use_phdf5=false;
 #endif
     }
-    Mode.set(IS_PARALLEL,use_pdf);
+    Mode.set(IS_PARALLEL,use_phdf5);
-    //true, if this task does not need to participate in I/O
+    Mode.set(IS_MASTER,!comm->rank());
-    if(use_collective)
+    if(request_pio&&!use_phdf5)
-      Mode.set(NOIO,comm->rank());
+      Mode.set(NOIO,comm->rank()); // master only
     else
-      Mode.set(NOIO,false);
+      Mode.set(NOIO,false); // pio or all.
   }
   else
   {
     Mode.set(IS_PARALLEL,false);
+    Mode.set(IS_MASTER,true);
     Mode.set(NOIO,false);
   }
 }
 
 bool hdf_archive::create(const std::string& fname, unsigned flags)
 {
-  //not I/O node, do nothing
   if(Mode[NOIO]) return true;
+  if(!(Mode[IS_PARALLEL]||Mode[IS_MASTER])) std::runtime_error("Only create file in parallel or by master but not every rank!");
   close(); 
   file_id = H5Fcreate(fname.c_str(),H5F_ACC_TRUNC,H5P_DEFAULT,access_id);
   return file_id != is_closed;
@@ -91,8 +99,7 @@ bool hdf_archive::create(const std::string& fname, unsigned flags)
 
 bool hdf_archive::open(const std::string& fname,unsigned flags)
 {
-  if(Mode[NOIO])
+  if(Mode[NOIO]) return true;
-    return true;
   close();
   file_id = H5Fopen(fname.c_str(),flags,access_id);
   return file_id != is_closed;
@@ -100,10 +107,8 @@ bool hdf_archive::open(const std::string& fname,unsigned flags)
 
 bool hdf_archive::is_group(const std::string& aname)
 {
-  if(Mode[NOIO])
+  if(Mode[NOIO]) return true;
-    return true;
+  if(file_id==is_closed) return false;
-  if(file_id==is_closed)
-    return false;
   hid_t p=group_id.empty()? file_id:group_id.top();
   p=(aname[0]=='/')?file_id:p;
   hid_t g=H5Gopen(p,aname.c_str());
@@ -115,8 +120,7 @@ bool hdf_archive::is_group(const std::string& aname)
 
 hid_t hdf_archive::push(const std::string& gname, bool createit)
 {
-  if(Mode[NOIO]||file_id==is_closed)
+  if(Mode[NOIO]||file_id==is_closed) return is_closed;
-    return is_closed;
   hid_t p=group_id.empty()? file_id:group_id.top();
   hid_t g=H5Gopen(p,gname.c_str());
   if(g<0 && createit)

src/io/hdf_archive.h

@@ -34,14 +34,16 @@ class Communicate;
 
 namespace qmcplusplus
 {
+
 /** class to handle hdf file
  */
 struct hdf_archive
 {
-  enum {IS_PARALLEL=0, NOIO};
+  enum {IS_PARALLEL=0, IS_MASTER, NOIO};
   static const hid_t is_closed=-1;
   /** bitset of the io mode
-   * Mode[IS_PARALLEL] : true, if collective
+   * Mode[IS_PARALLEL] : true, if parallel
+   * Mode[IS_MASTER] : true, if the node is master
    * Mode[NOIO] : true, if I/O is not performed
    */
   std::bitset<4> Mode;
@@ -59,19 +61,22 @@ struct hdf_archive
   std::stack<hid_t> group_id;
   /** constructor
    * @param c communicator
-   * @param use_collective turn on/off collective
+   * @param request_pio turns on parallel I/O,
+   *        if ture and PHDF5 is available, hdf_archive is in parallel collective IO mode
+   *        if ture and PHDF5 is not available, hdf_archive is in master-only IO mode
+   *        if false, hdf_archive is in independent IO mode
    */
-  hdf_archive(Communicate* c=0, bool use_collective=false);
+  hdf_archive(Communicate* c=0, bool request_pio=false);
   ///destructor
   ~hdf_archive();
 
   ///set the access property
-  void set_access_plist(bool use_collective, Communicate* comm);
+  void set_access_plist(bool request_pio, Communicate* comm);
 
-  ///return true if collective i/o
+  ///return true if parallel i/o
-  inline bool is_collective() const
+  inline bool is_parallel() const
   {
-    return Mode[0];
+    return Mode[IS_PARALLEL];
   }
 
   /** create a file
@@ -128,8 +133,8 @@ struct hdf_archive
 
   template<typename T> bool write(T& data, const std::string& aname)
   {
-    if(Mode[NOIO])
+    if(Mode[NOIO]) return true;
-      return true;
+    if(!(Mode[IS_PARALLEL]||Mode[IS_MASTER])) std::runtime_error("Only write data in parallel or by master but not every rank!");
     hid_t p=group_id.empty()? file_id:group_id.top();
     h5data_proxy<T> e(data);
     return e.write(p,aname,xfer_plist);
@@ -137,8 +142,7 @@ struct hdf_archive
 
   template<typename T> bool read(T& data, const std::string& aname)
   {
-    if(Mode[NOIO])
+    if(Mode[NOIO]) return true;
-      return true;
     hid_t p=group_id.empty()? file_id:group_id.top();
     h5data_proxy<T> e(data);
     return e.read(p,aname,xfer_plist);
@@ -146,8 +150,7 @@ struct hdf_archive
 
   inline void unlink(const std::string& aname)
   {
-    if(Mode[NOIO])
+    if(Mode[NOIO]) return;
-      return;
     hid_t p=group_id.empty()? file_id:group_id.top();
     herr_t status=H5Gunlink(p,aname.c_str());
   }

src/io/hdf_dataproxy.h

@@ -115,6 +115,7 @@ bool h5d_read(hid_t grp, const std::string& aname, hsize_t ndims,
   return ret != -1;
 }
 
+
 template<typename T>
 inline bool h5d_write(hid_t grp, const std::string& aname, hsize_t ndims, 
     const hsize_t* gcounts, const hsize_t* counts, const hsize_t* offsets,
@@ -124,22 +125,35 @@ inline bool h5d_write(hid_t grp, const std::string& aname, hsize_t ndims,
     return true;
   hid_t h5d_type_id=get_h5_datatype(*first);
   hid_t h1 = H5Dopen(grp, aname.c_str());
+  hid_t filespace, memspace;
   herr_t ret=-1;
   if(h1<0) //missing create one
   {
     hid_t dataspace=H5Screate_simple(ndims,gcounts,NULL);
     hid_t dataset=H5Dcreate(grp, aname.c_str(),h5d_type_id, dataspace, H5P_DEFAULT);
 
-    hid_t filespace=H5Dget_space(dataspace);
+    hid_t filespace=H5Dget_space(dataset);
     ret=H5Sselect_hyperslab(filespace,H5S_SELECT_SET,offsets,NULL,counts,NULL); 
 
     hid_t memspace=H5Screate_simple(ndims,counts,NULL);
     ret=H5Dwrite(dataset,h5d_type_id,memspace,filespace,xfer_plist,first);
 
+    H5Dclose(memspace);
     H5Sclose(filespace);
     H5Dclose(dataset);
     H5Sclose(dataspace);
   }
+  else
+  {
+    filespace=H5Dget_space(h1);
+    ret=H5Sselect_hyperslab(filespace,H5S_SELECT_SET,offsets,NULL,counts,NULL);
+
+    memspace=H5Screate_simple(ndims,counts,NULL);
+    ret = H5Dwrite(h1, h5d_type_id, memspace, filespace, xfer_plist, first);
+
+    H5Sclose(filespace);
+    H5Dclose(memspace);
+  }
   H5Dclose(h1);
   return ret != -1;
 }

src/io/hdf_hyperslab.h

@@ -4,9 +4,9 @@
 //
 // Copyright (c) 2016 Jeongnim Kim and QMCPACK developers.
 //
-// File developed by: Jeremy McMinnis, jmcminis@gmail.com, University of Illinois at Urbana-Champaign   
+// File developed by: Jeremy McMinnis, jmcminis@gmail.com, University of Illinois at Urbana-Champaign
 //
-// File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign 
+// File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
 //////////////////////////////////////////////////////////////////////////////////////
 
 
@@ -40,7 +40,7 @@ struct hyperslab_proxy: public container_proxy<CT>
   ///1D
   hyperslab_proxy(CT& a): container_proxy<CT>(a), slab_rank(a.slab_rank),
     slab_dims(a.slab_dims), slab_offset(a.slab_offset)
-  { 
+  {
     slab_dims_local=slab_dims;
     use_slab=false;
   }
@@ -105,7 +105,7 @@ struct h5data_proxy<hyperslab_proxy<CT,MAXDIM> >
     {
       return h5d_read(grp,aname.c_str(),
           ref_.slab_rank,
-          ref_.slab_dims.data(), 
+          ref_.slab_dims.data(),
           ref_.slab_dims_local.data(),
           ref_.slab_offset.data(),
           ref_.data(),xfer_plist);
@@ -122,18 +122,18 @@ struct h5data_proxy<hyperslab_proxy<CT,MAXDIM> >
   }
   inline bool write(hid_t grp, const std::string& aname, hid_t xfer_plist=H5P_DEFAULT)
   {
-    //if(ref_.use_slab)
+    if(ref_.use_slab)
-    //{
+    {
-    //  std::cout << "Everyone writes a part " << ref_.slab_dims_local[0] << std::endl;
+      return h5d_write(grp,aname.c_str(),
-    //  return h5d_write(grp,aname.c_str(),
+          ref_.slab_rank,
-    //      ref_.slab_rank,
+          ref_.slab_dims.data(),
-    //      ref_.slab_dims.data(), 
+          ref_.slab_dims_local.data(),
-    //      ref_.slab_dims_local.data(),
+          ref_.slab_offset.data(),
-    //      ref_.slab_offset.data(),
+          ref_.data(),xfer_plist);
-    //      ref_.data(),xfer_plist);
+    }
-    //}
+    else{
-    //else
     return h5d_write(grp,aname.c_str(),ref_.slab_rank,ref_.slab_dims.data(),ref_.data(),xfer_plist);
+    }
   }
 };
 }

src/miniapps/restart.cpp

@@ -26,6 +26,7 @@
 #include <Utilities/Timer.h>
 #include <miniapps/common.hpp>
 #include <getopt.h>
+#include <mpi/collectives.h>
 
 using namespace std;
 using namespace qmcplusplus;
@@ -49,12 +50,13 @@ int main(int argc, char** argv)
   Communicate* myComm=OHMMS::Controller;
   myComm->setName("restart");
 
-  typedef QMCTraits::RealType           RealType;
+  typedef QMCTraits::RealType              RealType;
-  typedef ParticleSet::ParticlePos_t    ParticlePos_t;
+  typedef ParticleSet::ParticlePos_t       ParticlePos_t;
-  typedef ParticleSet::ParticleLayout_t LatticeType;
+  typedef ParticleSet::ParticleLayout_t    LatticeType;
-  typedef ParticleSet::TensorType       TensorType;
+  typedef ParticleSet::TensorType          TensorType;
-  typedef ParticleSet::PosType          PosType;
+  typedef ParticleSet::PosType             PosType;
-  typedef RandomGenerator_t::uint_type  uint_type;
+  typedef RandomGenerator_t::uint_type     uint_type;
+  typedef MCWalkerConfiguration::Walker_t  Walker_t;
 
   //use the global generator
 
@@ -100,9 +102,8 @@ int main(int argc, char** argv)
   // set the number of walkers equal to the threads.
   if(!AverageWalkersPerNode) AverageWalkersPerNode=NumThreads;
   //set nwtot, to be random
-  nwtot=AverageWalkersPerNode;
+  nwtot=std::abs(AverageWalkersPerNode+myComm->rank()%5-2);
   FairDivideLow(nwtot,NumThreads,wPerNode);
-  wPerNode.resize(NumThreads+1,0);
 
   //Random.init(0,1,iseed);
   Tensor<int,3> tmat(na,0,0,0,nb,0,0,0,nc);
@@ -119,22 +120,23 @@ int main(int argc, char** argv)
 
   RandomNumberControl::make_seeds();
   std::vector<RandomGenerator_t> myRNG(NumThreads);
+  std::vector<uint_type> mt(Random.state_size(),0);
   std::vector<MCWalkerConfiguration> elecs(NumThreads);
 
+  ParticleSet ions;
+  OHMMS_PRECISION scale=1.0;
+  tile_graphite(ions,tmat,scale);
+
   #pragma omp parallel reduction(+:t0)
   {
     int ip=omp_get_thread_num();
 
-    ParticleSet ions;
     MCWalkerConfiguration& els=elecs[ip];
-    OHMMS_PRECISION scale=1.0;
 
     //create generator within the thread
     myRNG[ip]=*RandomNumberControl::Children[ip];
     RandomGenerator_t& random_th=myRNG[ip];
 
-    tile_graphite(ions,tmat,scale);
-
     const int nions=ions.getTotalNum();
     const int nels=4*nions;
     const int nels3=3*nels;
@@ -152,18 +154,33 @@ int main(int argc, char** argv)
       els.RSoA=els.R;
     }
 
+    if(!ip) elecs[0].createWalkers(nwtot);
+    #pragma omp barrier
+
+    for(MCWalkerConfiguration::iterator wi=elecs[0].begin()+wPerNode[ip]; wi!=elecs[0].begin()+wPerNode[ip+1]; wi++)
+      els.saveWalker(**wi);
+
     // save random seeds and electron configurations.
     *RandomNumberControl::Children[ip]=myRNG[ip];
     //MCWalkerConfiguration els_save(els);
 
   } //end of omp parallel
+  Random.save(mt);
 
-  elecs[0].createWalkers(nwtot);
   setWalkerOffsets(elecs[0], myComm);
 
+  //storage variables for timers
+  double h5write = 0.0, h5read = 0.0; //random seed R/W speeds
+  double walkerWrite = 0.0, walkerRead =0.0; //walker R/W speeds
+  Timer h5clock; //timer for the program
+
   // dump random seeds
+  myComm->barrier();
+  h5clock.restart(); //start timer
   RandomNumberControl::write("restart",myComm);
   myComm->barrier();
+  h5write += h5clock.elapsed(); //store timer
+
   // flush random seeds to zero
   #pragma omp parallel
   {
@@ -172,9 +189,15 @@ int main(int argc, char** argv)
     std::vector<uint_type> vt(random_th.state_size(),0);
     random_th.load(vt);
   }
+  std::vector<uint_type> mt_temp(Random.state_size(),0);
+  Random.load(mt_temp);
 
   // load random seeds
+  myComm->barrier();
+  h5clock.restart(); //start timer
   RandomNumberControl::read("restart",myComm);
+  myComm->barrier();
+  h5read += h5clock.elapsed(); //store timer
 
   // validate random seeds
   int mismatch_count=0;
@@ -189,6 +212,9 @@ int main(int argc, char** argv)
     for(int i=0; i<random_th.state_size(); i++)
       if(vt_orig[i]!=vt_load[i]) mismatch_count++;
   }
+  Random.save(mt_temp);
+  for(int i=0; i<Random.state_size(); i++)
+    if(mt_temp[i]!=mt[i]) mismatch_count++;
 
   myComm->allreduce(mismatch_count);
 
@@ -196,16 +222,27 @@ int main(int argc, char** argv)
   {
     if(mismatch_count!=0)
       std::cout << "Fail: random seeds mismatch between write and read!\n"
-                << "state_size= " << myRNG[0].state_size() << " mismatch_cout=" << mismatch_count << std::endl;
+                << "  state_size= " << myRNG[0].state_size() << " mismatch_cout=" << mismatch_count << std::endl;
     else
       std::cout << "Pass: random seeds match exactly between write and read!\n";
   }
 
   // dump electron coordinates.
   HDFWalkerOutput wOut(elecs[0],"restart",myComm);
+  myComm->barrier();
+  h5clock.restart(); //start timer
   wOut.dump(elecs[0],1);
   myComm->barrier();
+  walkerWrite += h5clock.elapsed(); //store timer
+  if(!myComm->rank()) std::cout << "Walkers are dumped!\n";
 
+  // save walkers before destroying them
+  std::vector<Walker_t> saved_walkers;
+  for(int wi=0; wi<elecs[0].getActiveWalkers(); wi++)
+    saved_walkers.push_back(*elecs[0][wi]);
+  elecs[0].destroyWalkers(elecs[0].begin(),elecs[0].end());
+
+  // load walkers
   const char *restart_input = \
 "<tmp> \
   <mcwalkerset fileroot=\"restart\" node=\"-1\" version=\"3 0\" collected=\"yes\"/> \
@@ -219,8 +256,68 @@ int main(int argc, char** argv)
 
   HDFVersion in_version(0,4);
   HDFWalkerInput_0_4 wIn(elecs[0],myComm,in_version);
+  myComm->barrier();
+  h5clock.restart(); //start timer
   wIn.put(restart_leaf);
-  
+  myComm->barrier();
+  walkerRead += h5clock.elapsed(); //store time spent
+
+  if(saved_walkers.size()!=elecs[0].getActiveWalkers())
+    std::cout << "Fail: Rank " << myComm->rank() << " had " << saved_walkers.size()
+              << "  walkers but loaded only " << elecs[0].getActiveWalkers() << " walkers from file!" << std::endl;
+
+  mismatch_count=0;
+  for(int wi=0; wi<saved_walkers.size(); wi++)
+  {
+    saved_walkers[wi].R=saved_walkers[wi].R-elecs[0][wi]->R;
+    if(Dot(saved_walkers[wi].R,saved_walkers[wi].R)>std::numeric_limits<RealType>::epsilon()) mismatch_count++;
+  }
+  myComm->allreduce(mismatch_count);
+
+  if(!myComm->rank())
+  {
+    if(mismatch_count!=0)
+      std::cout << "Fail: electron coordinates mismatch between write and read!\n"
+                << "  mismatch_cout=" << mismatch_count << std::endl;
+    else
+      std::cout << "Pass: electron coordinates match exactly between write and read!\n";
+  }
+
+  //print out hdf5 R/W times
+  TinyVector<double,4> timers(h5read, h5write, walkerRead, walkerWrite);
+  mpi::reduce(*myComm, timers);
+  h5read = timers[0]/myComm->size();
+  h5write = timers[1]/myComm->size();
+  walkerRead = timers[2]/myComm->size();
+  walkerWrite = timers[3]/myComm->size();
+  if(myComm->rank() == 0)
+  {
+    cout << "\nTotal time of writing random seeds to HDF5 file: " << setprecision(2) << h5write << "\n";
+    cout << "\nTotal time of reading random seeds in HDF5 file: " << setprecision(2) << h5read << "\n";
+    cout << "\nTotal time of writing walkers to HDF5 file: " << setprecision(2) << walkerWrite << "\n";
+    cout << "\nTotal time of reading walkers in HDF5 file: " << setprecision(2) << walkerRead << "\n";
+  }
+
+  if(myComm->size()>1)
+  {
+    Communicate* subComm = new Communicate(*myComm, 2);
+    subComm->setName("restart2");
+
+    if(subComm->getGroupID() == 0)
+    {
+      elecs[0].destroyWalkers(elecs[0].begin(),elecs[0].end());
+      HDFWalkerInput_0_4 subwIn(elecs[0],subComm,in_version);
+      subwIn.put(restart_leaf);
+      subComm->barrier();
+      if(!subComm->rank()) std::cout << "Walkers are loaded again by the subgroup!\n";
+      setWalkerOffsets(elecs[0], subComm);
+      HDFWalkerOutput subwOut(elecs[0],"XXXX",subComm);
+      subwOut.dump(elecs[0],1);
+      if(!subComm->rank()) std::cout << "Walkers are dumped again by the subgroup!\n";
+    }
+    delete subComm;
+  }
+
   OHMMS::Controller->finalize();
 
   return 0;