mirror of https://github.com/QMCPACK/qmcpack.git
commit
ad11debd9b
|
@ -231,7 +231,7 @@ SET(OHMMS_ORTHO 0)
|
|||
#list of options to determine libraries. First, try to use libraries available
|
||||
#FindXYZ.cmake will set the values if successful
|
||||
######################################################################
|
||||
SET(ENABLE_PHDF5 0 CACHE BOOL "Enable phdf5 for output")
|
||||
SET(ENABLE_PHDF5 1 CACHE BOOL "Enable phdf5 for output")
|
||||
SET(HAVE_LIBXML2 1)
|
||||
SET(HAVE_LIBFFTW 0)
|
||||
SET(HAVE_LIBXMLPP 0)
|
||||
|
@ -654,6 +654,20 @@ IF(HDF5_FOUND)
|
|||
#IF(SZLIB_FOUND)
|
||||
# SET(QMC_UTIL_LIBS ${QMC_UTIL_LIBS} ${SZLIB_LIBRARIES})
|
||||
#ENDIF(SZLIB_FOUND)
|
||||
IF(HDF5_IS_PARALLEL)
|
||||
IF(ENABLE_PHDF5)
|
||||
MESSAGE(STATUS "Using HDF5 parallel collective I/O")
|
||||
ELSE(ENABLE_PHDF5)
|
||||
MESSAGE(STATUS "Using HDF5 non-scalable serial I/O")
|
||||
ENDIF(ENABLE_PHDF5)
|
||||
ELSE(HDF5_IS_PARALLEL)
|
||||
IF(ENABLE_PHDF5)
|
||||
MESSAGE(STATUS "Using HDF5 non-scalable serial I/O due to the lack of library support for parallel")
|
||||
SET(ENABLE_PHDF5 0)
|
||||
ELSE(ENABLE_PHDF5)
|
||||
MESSAGE(STATUS "Using HDF5 non-scalable serial I/O")
|
||||
ENDIF(ENABLE_PHDF5)
|
||||
ENDIF(HDF5_IS_PARALLEL)
|
||||
ELSE(HDF5_FOUND)
|
||||
if(NOT QMC_PHI)
|
||||
MESSAGE(FATAL_ERROR "Require hdf5 1.6.4 or higher. Set HDF5_ROOT")
|
||||
|
|
|
@ -28,7 +28,7 @@ SET(BOOST_ROOT /soft/libraries/boost/1.62.0/cnk-bgclang++11/current/)
|
|||
|
||||
SET(CMAKE_FIND_ROOT_PATH
|
||||
/home/projects/qmcpack/libXML2-2.9.1
|
||||
/soft/libraries/hdf5/1.8.14/cnk-gcc/current
|
||||
/soft/libraries/hdf5/current/cnk-gcc/current
|
||||
/soft/libraries/alcf/current/gcc/FFTW3
|
||||
/soft/libraries/alcf/current/gcc/ZLIB
|
||||
)
|
||||
|
|
|
@ -12,8 +12,6 @@
|
|||
//
|
||||
// File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
|
||||
#include <Configuration.h>
|
||||
|
@ -239,7 +237,7 @@ bool RandomNumberControl::put(xmlNodePtr cur)
|
|||
return true;
|
||||
}
|
||||
|
||||
void RandomNumberControl::read(const std::string& fname, Communicate* comm)
|
||||
void RandomNumberControl::read_old(const std::string& fname, Communicate* comm)
|
||||
{
|
||||
int nthreads=omp_get_max_threads();
|
||||
std::vector<uint_type> vt_tot, vt;
|
||||
|
@ -321,7 +319,7 @@ void RandomNumberControl::read(const std::string& fname, Communicate* comm)
|
|||
}
|
||||
}
|
||||
|
||||
void RandomNumberControl::write(const std::string& fname, Communicate* comm)
|
||||
void RandomNumberControl::write_old(const std::string& fname, Communicate* comm)
|
||||
{
|
||||
int nthreads=omp_get_max_threads();
|
||||
std::vector<uint_type> vt, vt_tot;
|
||||
|
@ -375,4 +373,238 @@ void RandomNumberControl::write(const std::string& fname, Communicate* comm)
|
|||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/*New functions past this point*/
|
||||
//switch between read functions
|
||||
void RandomNumberControl::read(const std::string& fname, Communicate* comm)
|
||||
{
|
||||
std::string h5name=fname+".random.h5";
|
||||
hdf_archive hin(comm, true); //attempt to read in parallel
|
||||
hin.open(h5name,H5F_ACC_RDONLY);
|
||||
if(hin.is_parallel())
|
||||
read_parallel(hin, comm);
|
||||
else
|
||||
read_rank_0(hin, comm);
|
||||
}
|
||||
|
||||
//switch between write functions
|
||||
void RandomNumberControl::write(const std::string& fname, Communicate* comm)
|
||||
{
|
||||
std::string h5name=fname+".random.h5";
|
||||
hdf_archive hout(comm, true); //attempt to write in parallel
|
||||
hout.create(h5name);
|
||||
if(hout.is_parallel())
|
||||
write_parallel(hout, comm);
|
||||
else
|
||||
write_rank_0(hout, comm);
|
||||
}
|
||||
|
||||
//Parallel read
|
||||
void RandomNumberControl::read_parallel(hdf_archive& hin, Communicate* comm)
|
||||
{
|
||||
int nthreads = omp_get_max_threads();
|
||||
std::vector<uint_type> vt, mt;
|
||||
TinyVector<int,3> shape_now(comm->size(), nthreads, Random.state_size()); //cur configuration
|
||||
TinyVector<int,3> shape_hdf5(3,0); //configuration when file was written
|
||||
|
||||
//grab shape and Random.state_size() used to create hdf5 file
|
||||
hin.push(hdf::main_state);
|
||||
hin.read(shape_hdf5, "nprocs_nthreads_statesize");
|
||||
|
||||
//if hdf5 file's shape and the current shape don't match, abort read
|
||||
if(shape_hdf5[0] != shape_now[0] || shape_hdf5[1] != shape_now[1] || shape_hdf5[2] != shape_now[2])
|
||||
{
|
||||
app_log() << "Mismatched random number generators."
|
||||
<< "\n Number of procs in streams : old=" << shape_hdf5[0] << " new= " << shape_now[0]
|
||||
<< "\n Number of threads in streams : old=" << shape_hdf5[1] << " new= " << shape_now[1]
|
||||
<< "\n State size per stream : old=" << shape_hdf5[2] << " new= " << shape_now[2]
|
||||
<< "\n Using the random streams generated at the initialization.\n";
|
||||
return;
|
||||
}
|
||||
app_log() << " Restart from the random number streams from the previous configuration.\n";
|
||||
|
||||
TinyVector<int,2> shape(comm->size()*nthreads, Random.state_size()); //global dims of children dataset
|
||||
vt.resize(nthreads*Random.state_size()); //buffer for children[ip]
|
||||
mt.resize(Random.state_size()); //buffer for single thread Random object of random nums
|
||||
|
||||
TinyVector<int,2> counts(nthreads, Random.state_size()); //local dimensions of dataset
|
||||
TinyVector<int,2> offsets(comm->rank() * nthreads, 0); //offsets for each process to read in
|
||||
|
||||
hin.push("random"); //group that holds children[ip] random nums
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab(vt, shape, counts, offsets);
|
||||
hin.read(slab,Random.EngineName);
|
||||
|
||||
hin.pop();
|
||||
hin.push("random_master"); //group that holds Random_th random nums
|
||||
shape[0] = comm->size(); //reset shape, counts and offset for non-multiple threads
|
||||
counts[0] = 1;
|
||||
offsets[0] = comm->rank();
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab2(mt, shape, counts, offsets);
|
||||
hin.read(slab2,Random.EngineName);
|
||||
hin.close();
|
||||
|
||||
std::vector<uint_type>::iterator vt_it(vt.begin());
|
||||
for(int ip=0; ip<nthreads; ip++, vt_it += shape[1])
|
||||
{
|
||||
std::vector<uint_type> c(vt_it,vt_it+shape[1]);
|
||||
Children[ip]->load(c); //load random nums back to program from buffer
|
||||
}
|
||||
Random.load(mt); //load random nums back to prog from buffer
|
||||
}
|
||||
|
||||
//Parallel write
|
||||
void RandomNumberControl::write_parallel(hdf_archive& hout, Communicate* comm)
|
||||
{
|
||||
int nthreads=omp_get_max_threads();
|
||||
std::vector<uint_type> vt, mt;
|
||||
TinyVector<int,3> shape_hdf5(comm->size(), nthreads, Random.state_size()); //configuration at write time
|
||||
vt.reserve(nthreads*Random.state_size()); //buffer for random numbers from children[ip] of each thread
|
||||
mt.reserve(Random.state_size()); //buffer for random numbers from single Random object
|
||||
|
||||
for(int ip=0; ip<nthreads; ++ip)
|
||||
{
|
||||
std::vector<uint_type> c;
|
||||
Children[ip]->save(c);
|
||||
vt.insert(vt.end(),c.begin(),c.end()); //get nums from each thread into buffer
|
||||
}
|
||||
Random.save(mt); //get nums for single random object (no threads)
|
||||
|
||||
TinyVector<int,2> shape(comm->size()*nthreads,Random.state_size()); //global dimensions
|
||||
TinyVector<int,2> counts(nthreads, Random.state_size()); //local dimensions
|
||||
TinyVector<int,2> offsets(comm->rank() * nthreads, 0); //offset for the file write
|
||||
|
||||
hout.push(hdf::main_state);
|
||||
hout.write(shape_hdf5, "nprocs_nthreads_statesize"); //save the shape of the data at write
|
||||
|
||||
hout.push("random"); //group for children[ip]
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab(vt, shape, counts, offsets);
|
||||
hout.write(slab,Random.EngineName); //write to hdf5file
|
||||
hout.pop();
|
||||
|
||||
shape[0] = comm->size(); //adjust shape, counts, offset for just one thread
|
||||
counts[0] = 1;
|
||||
offsets[0] = comm->rank();
|
||||
hout.push("random_master"); //group for random object without threads
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab2(mt, shape, counts, offsets);
|
||||
hout.write(slab2,Random.EngineName); //write data to hdf5 file
|
||||
hout.close();
|
||||
}
|
||||
|
||||
//Scatter read
|
||||
void RandomNumberControl::read_rank_0(hdf_archive& hin, Communicate* comm)
|
||||
{
|
||||
int nthreads = omp_get_max_threads();
|
||||
std::vector<uint_type> vt, vt_tot, mt, mt_tot;
|
||||
TinyVector<int,3> shape_now(comm->size(), nthreads, Random.state_size()); //current configuration
|
||||
TinyVector<int,2> shape(comm->size()*nthreads, Random.state_size()); //dimensions of children dataset
|
||||
TinyVector<int,3> shape_hdf5(3,0); //configuration when hdf5 file was written
|
||||
|
||||
//grab configuration of threads/procs and Random.state_size() in hdf5 file
|
||||
if(comm->rank() == 0)
|
||||
{
|
||||
hin.push(hdf::main_state);
|
||||
hin.read(shape_hdf5, "nprocs_nthreads_statesize");
|
||||
}
|
||||
|
||||
mpi::bcast(*comm, shape_hdf5);
|
||||
|
||||
//if hdf5 file's configuration and current configuration don't match, abort read
|
||||
if(shape_hdf5[0] != shape_now[0] || shape_hdf5[1] != shape_now[1] || shape_hdf5[2] != shape_now[2])
|
||||
{
|
||||
app_log() << "Mismatched random number generators."
|
||||
<< "\n Number of procs in streams : old=" << shape_hdf5[0] << " new= " << shape_now[0]
|
||||
<< "\n Number of threads in streams : old=" << shape_hdf5[1] << " new= " << shape_now[1]
|
||||
<< "\n State size per stream : old=" << shape_hdf5[2] << " new= " << shape_now[2]
|
||||
<< "\n Using the random streams generated at the initialization.\n";
|
||||
return;
|
||||
}
|
||||
app_log() << " Restart from the random number streams from the previous configuration.\n";
|
||||
|
||||
vt.resize(nthreads*Random.state_size()); //buffer for random nums in children of each thread
|
||||
mt.resize(Random.state_size()); //buffer for random numbers from single Random object
|
||||
|
||||
if(comm->rank() == 0)
|
||||
{
|
||||
hin.push("random"); //group for children[ip] (Random.object for each thread)
|
||||
vt_tot.resize(nthreads*Random.state_size()*comm->size());
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab(vt_tot, shape);
|
||||
hin.read(slab,Random.EngineName);
|
||||
hin.pop();
|
||||
|
||||
shape[0] = comm->size(); //reset shape to one thread per process
|
||||
mt_tot.resize(Random.state_size()*comm->size());
|
||||
hin.push("random_master"); //group for single Random object
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab2(mt_tot, shape);
|
||||
hin.read(slab2,Random.EngineName);
|
||||
hin.close();
|
||||
}
|
||||
|
||||
if(comm->size()>1)
|
||||
{
|
||||
mpi::scatter(*comm,vt_tot,vt); //divide big buffer into on for each proc
|
||||
mpi::scatter(*comm,mt_tot,mt);
|
||||
}
|
||||
else
|
||||
{
|
||||
copy(vt_tot.begin(),vt_tot.end(),vt.begin());
|
||||
copy(mt_tot.begin(),mt_tot.end(),mt.begin());
|
||||
}
|
||||
|
||||
std::vector<uint_type>::iterator vt_it(vt.begin());
|
||||
for(int i=0; i<nthreads; i++, vt_it += shape[1])
|
||||
{
|
||||
std::vector<uint_type> c(vt_it,vt_it+shape[1]);
|
||||
Children[i]->load(c); //read seeds for each thread from buffer back into object
|
||||
}
|
||||
Random.load(mt); //read seeds back into object
|
||||
}
|
||||
|
||||
//scatter write
|
||||
void RandomNumberControl::write_rank_0(hdf_archive& hout, Communicate* comm)
|
||||
{
|
||||
int nthreads = omp_get_max_threads();
|
||||
std::vector<uint_type> vt, vt_tot, mt, mt_tot;
|
||||
TinyVector<int,2> shape(comm->size()*nthreads, Random.state_size()); //dimensions of children dataset
|
||||
TinyVector<int,3> shape_hdf5(comm->size(), nthreads, Random.state_size()); //configuration at write time
|
||||
vt.reserve(nthreads*Random.state_size()); //buffer for children[ip] (Random object of seeds for each thread)
|
||||
mt.reserve(Random.state_size()); //buffer for single Random object of seeds, one per proc regardless of thread num
|
||||
|
||||
for(int i=0; i<nthreads; ++i)
|
||||
{
|
||||
std::vector<uint_type> c;
|
||||
Children[i]->save(c);
|
||||
vt.insert(vt.end(),c.begin(),c.end()); //copy children[nthreads] seeds to buffer
|
||||
}
|
||||
Random.save(mt); //copy random_th seeds to buffer
|
||||
|
||||
if(comm->size()>1)
|
||||
{
|
||||
vt_tot.resize(vt.size()*comm->size());
|
||||
mt_tot.resize(mt.size()*comm->size());
|
||||
mpi::gather(*comm,vt,vt_tot); //gather into one big buffer for master write
|
||||
mpi::gather(*comm,mt,mt_tot);
|
||||
}
|
||||
else
|
||||
{
|
||||
vt_tot=vt;
|
||||
mt_tot=mt;
|
||||
}
|
||||
|
||||
if(comm->rank()==0)
|
||||
{
|
||||
hout.push(hdf::main_state);
|
||||
hout.write(shape_hdf5, "nprocs_nthreads_statesize"); //configuration at write time to file
|
||||
|
||||
hout.push("random"); //group for children[ip]
|
||||
hyperslab_proxy<std::vector<uint_type>, 2> slab(vt_tot, shape);
|
||||
hout.write(slab, Random.EngineName);
|
||||
hout.pop();
|
||||
|
||||
shape[0] = comm->size(); //reset dims for single thread use
|
||||
hout.push("random_master"); //group for random_th object
|
||||
hyperslab_proxy<std::vector<uint_type>,2> slab2(mt_tot, shape);
|
||||
hout.write(slab2,Random.EngineName);
|
||||
hout.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -10,8 +10,6 @@
|
|||
//
|
||||
// File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
|
||||
#ifndef OHMMS_RANDOMNUMBERCONTROL_H__
|
||||
|
@ -19,6 +17,7 @@
|
|||
#include "OhmmsData/OhmmsElementBase.h"
|
||||
#include "Utilities/RandomGenerator.h"
|
||||
#include "Utilities/PrimeNumberSet.h"
|
||||
#include <io/hdf_archive.h>
|
||||
|
||||
class Communicate;
|
||||
|
||||
|
@ -56,17 +55,48 @@ public:
|
|||
|
||||
xmlNodePtr initialize(xmlXPathContextPtr);
|
||||
|
||||
/** read random state from a hdf file
|
||||
/** read in parallel or serial
|
||||
* @param fname file name
|
||||
* @param comm communicator so that everyone reads its own data
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void read(const std::string& fname, Communicate* comm);
|
||||
/** write random state to a hdf file
|
||||
/** write in parallel or serial
|
||||
* @param fname file name
|
||||
* @param comm communicator so that everyone writes its own data
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void write(const std::string& fname, Communicate* comm);
|
||||
|
||||
/** read random state from a hdf file in parallel
|
||||
* @param hdf_archive set to parallel
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void read_parallel(hdf_archive& hin, Communicate* comm);
|
||||
/** write random state to a hdf file in parallel
|
||||
* @param hdf_archive set to parallel
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void write_parallel(hdf_archive& hout, Communicate* comm);
|
||||
/** rank 0 reads random states from a hdf file
|
||||
* and distributes them to all the other ranks
|
||||
* @param hdf_archive set to serial
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void read_rank_0(hdf_archive& hin, Communicate* comm);
|
||||
/** rank 0 gathers the random states from all the other ranks
|
||||
* and write them to a hdf file
|
||||
* @param hin hdf_archive object set to serial
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void write_rank_0(hdf_archive& hout, Communicate* comm);
|
||||
/** read random state from a xml file
|
||||
* @param fname file name
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void read_old(const std::string& fname, Communicate* comm);
|
||||
/** write random state to a xml file
|
||||
* @param fname file name
|
||||
* @param comm communicator
|
||||
*/
|
||||
static void write_old(const std::string& fname, Communicate* comm);
|
||||
private:
|
||||
|
||||
bool NeverBeenInitialized;
|
||||
|
|
|
@ -126,7 +126,11 @@ bool HDFWalkerInput_0_4::put(xmlNodePtr cur)
|
|||
FileStack.pop();
|
||||
std::string h5name(FileName);
|
||||
//success |= read_hdf5_scatter(h5name);
|
||||
#ifdef ENABLE_PHDF5
|
||||
success |= read_phdf5(h5name);
|
||||
#else
|
||||
success |= read_hdf5(h5name);
|
||||
#endif
|
||||
}
|
||||
return success;
|
||||
#endif
|
||||
|
@ -286,55 +290,68 @@ bool HDFWalkerInput_0_4::read_hdf5_scatter( std::string h5name)
|
|||
|
||||
bool HDFWalkerInput_0_4::read_phdf5( std::string h5name)
|
||||
{
|
||||
//Broken
|
||||
//#if defined(H5_HAVE_PARALLEL) && defined(ENABLE_PHDF5)
|
||||
// hin.read(aversion,hdf::version);
|
||||
// int found_group=hin.is_group(hdf::main_state);
|
||||
// if(!found_group)
|
||||
// continue;
|
||||
// //start main_state
|
||||
// hin.push(hdf::main_state);
|
||||
// int nw_in=0;
|
||||
// hin.read(nw_in,hdf::num_walkers);
|
||||
// std::vector<int> woffsets(myComm->size()+1,0);
|
||||
// FairDivideLow(nw_in,myComm->size(),woffsets);
|
||||
// int mynode=myComm->rank();
|
||||
// int nw_loc=woffsets[mynode+1]-woffsets[mynode];
|
||||
// std::cout << "node =" << mynode << " nw_loc=" << nw_loc << " nw_in=" << nw_in << std::endl;
|
||||
// //TinyVector<hsize_t,3> gcounts, counts, offset;
|
||||
// hsize_t gcounts[3], counts[3], offset[3];
|
||||
// gcounts[0]=nw_in;
|
||||
// gcounts[1]=targetW.getTotalNum();
|
||||
// gcounts[2]=OHMMS_DIM;
|
||||
// counts[0]=nw_loc;
|
||||
// counts[1]=targetW.getTotalNum();
|
||||
// counts[2]=OHMMS_DIM;
|
||||
// offset[0]=woffsets[mynode];
|
||||
// offset[1]=0;
|
||||
// offset[2]=0;
|
||||
// int nitems=targetW.getTotalNum()*OHMMS_DIM;
|
||||
// typedef std::vector<QMCTraits::RealType> Buffer_t;
|
||||
// Buffer_t posin(nw_loc*nitems);
|
||||
// hid_t dataset = H5Dopen(hin.top(),hdf::walkers);
|
||||
// hid_t dataspace = H5Dget_space(dataset);
|
||||
// int rank_n = H5Sget_simple_extent_ndims(dataspace);
|
||||
// int status_n = H5Sget_simple_extent_dims(dataspace, gcounts, NULL);
|
||||
// hid_t type_id=get_h5_datatype(posin[0]);
|
||||
// hid_t memspace = H5Screate_simple(3, counts, NULL);
|
||||
// herr_t status = H5Sselect_hyperslab(dataspace,H5S_SELECT_SET, offset,NULL,counts,NULL);
|
||||
// status = H5Dread(dataset, type_id, memspace, dataspace, hin.xfer_plist, &posin[0]);
|
||||
// H5Sclose(dataspace);
|
||||
// H5Sclose(memspace);
|
||||
// H5Dclose(dataset);
|
||||
// int curWalker = targetW.getActiveWalkers();
|
||||
// targetW.createWalkers(nw_loc);
|
||||
// Buffer_t::iterator it(posin.begin());
|
||||
// for(int i=0,iw=curWalker; i<nw_loc; ++i,++iw)
|
||||
// {
|
||||
// copy(it,it+nitems,get_first_address(targetW[iw]->R));
|
||||
// it += nitems;
|
||||
// }
|
||||
//#endif
|
||||
|
||||
int nw_in=0;
|
||||
h5name.append(hdf::config_ext);
|
||||
hdf_archive hin(myComm,true); //everone reads this
|
||||
bool success=hin.open(h5name,H5F_ACC_RDONLY);
|
||||
//check if hdf and xml versions can work together
|
||||
HDFVersion aversion;
|
||||
|
||||
hin.read(aversion,hdf::version);
|
||||
if(!(aversion < i_info.version))
|
||||
{
|
||||
int found_group=hin.is_group(hdf::main_state);
|
||||
hin.push(hdf::main_state);
|
||||
hin.read(nw_in,hdf::num_walkers);
|
||||
}
|
||||
else
|
||||
{
|
||||
app_error() << " Mismatched version. xml = " << i_info.version << " hdf = " << aversion << std::endl;
|
||||
}
|
||||
|
||||
if(nw_in==0)
|
||||
{
|
||||
app_error() << " No walkers in " << h5name << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
typedef std::vector<QMCTraits::RealType> Buffer_t;
|
||||
Buffer_t posin;
|
||||
TinyVector<int,3> dims(nw_in,targetW.getTotalNum(), OHMMS_DIM);
|
||||
|
||||
std::vector<int> woffsets;
|
||||
hin.read(woffsets,"walker_partition");
|
||||
int nw_loc=woffsets[myComm->rank()+1]-woffsets[myComm->rank()];
|
||||
|
||||
int np1=myComm->size()+1;
|
||||
if(woffsets.size()!= np1)
|
||||
{
|
||||
woffsets.resize(myComm->size()+1,0);
|
||||
FairDivideLow(nw_in,myComm->size(),woffsets);
|
||||
nw_loc=woffsets[myComm->rank()+1]-woffsets[myComm->rank()];
|
||||
}
|
||||
|
||||
TinyVector<int,3> counts(nw_loc, targetW.getTotalNum(), OHMMS_DIM);
|
||||
TinyVector<int,3> offsets(woffsets[myComm->rank()], 0, 0);
|
||||
posin.resize(nw_loc*dims[1]*dims[2]);
|
||||
|
||||
hyperslab_proxy<Buffer_t,3> slab(posin, dims, counts, offsets);
|
||||
hin.read(slab,hdf::walkers);
|
||||
|
||||
app_log() << " HDFWalkerInput_0_4::put getting " << dims[0] << " walkers " << posin.size() << std::endl;
|
||||
nw_in=woffsets[myComm->rank()+1]-woffsets[myComm->rank()];
|
||||
{
|
||||
int nitems=targetW.getTotalNum()*OHMMS_DIM;
|
||||
int curWalker = targetW.getActiveWalkers();
|
||||
targetW.createWalkers(nw_in);
|
||||
Buffer_t::iterator it(posin.begin());
|
||||
for(int i=0,iw=curWalker; i<nw_in; ++i,++iw)
|
||||
{
|
||||
copy(it,it+nitems,get_first_address(targetW[iw]->R));
|
||||
it += nitems;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool HDFWalkerInput_0_4::read_adios(xmlNodePtr cur)
|
||||
|
|
|
@ -202,15 +202,11 @@ void HDFWalkerOutput::write_configuration(MCWalkerConfiguration& W, hdf_archive&
|
|||
hout.write(number_of_walkers,hdf::num_walkers);
|
||||
|
||||
TinyVector<int,3> gcounts(number_of_walkers,number_of_particles,OHMMS_DIM);
|
||||
//vector<int> gcounts(3);
|
||||
//gcounts[0]=number_of_walkers;
|
||||
//gcounts[1]=number_of_particles;
|
||||
//gcounts[2]=OHMMS_DIM;
|
||||
|
||||
if(hout.is_collective())
|
||||
if(hout.is_parallel())
|
||||
{
|
||||
TinyVector<int,3> counts(W.getActiveWalkers(), number_of_particles,OHMMS_DIM);
|
||||
TinyVector<int,3> offsets(W.WalkerOffsets[myComm->rank()],number_of_particles,OHMMS_DIM);
|
||||
TinyVector<int,3> offsets(W.WalkerOffsets[myComm->rank()],0,0);
|
||||
hyperslab_proxy<BufferType,3> slab(*RemoteData[0],gcounts,counts,offsets);
|
||||
hout.write(slab,hdf::walkers);
|
||||
}
|
||||
|
|
|
@ -16,17 +16,17 @@
|
|||
namespace qmcplusplus
|
||||
{
|
||||
//const hid_t hdf_archive::is_closed;
|
||||
hdf_archive::hdf_archive(Communicate* c, bool use_collective)
|
||||
hdf_archive::hdf_archive(Communicate* c, bool request_pio)
|
||||
: file_id(is_closed), access_id(H5P_DEFAULT), xfer_plist(H5P_DEFAULT)
|
||||
{
|
||||
H5Eget_auto (&err_func, &client_data);
|
||||
H5Eset_auto (NULL, NULL);
|
||||
set_access_plist(use_collective,c);
|
||||
set_access_plist(request_pio,c);
|
||||
}
|
||||
|
||||
hdf_archive::~hdf_archive()
|
||||
{
|
||||
#if defined(H5_HAVE_PARALLEL) && defined(ENABLE_PHDF5)
|
||||
#if defined(ENABLE_PHDF5)
|
||||
if(xfer_plist != H5P_DEFAULT) H5Pclose(xfer_plist);
|
||||
if(access_id != H5P_DEFAULT) H5Pclose(access_id);
|
||||
#endif
|
||||
|
@ -48,42 +48,50 @@ void hdf_archive::close()
|
|||
file_id=is_closed;
|
||||
}
|
||||
|
||||
void hdf_archive::set_access_plist(bool use_collective, Communicate* comm)
|
||||
void hdf_archive::set_access_plist(bool request_pio, Communicate* comm)
|
||||
{
|
||||
access_id=H5P_DEFAULT;
|
||||
if(comm && comm->size()>1) //for parallel communicator
|
||||
{
|
||||
bool use_pdf=false;
|
||||
if(use_collective)
|
||||
bool use_phdf5=false;
|
||||
if(request_pio)
|
||||
{
|
||||
#if defined(H5_HAVE_PARALLEL) && defined(ENABLE_PHDF5)
|
||||
// MPI_Info info=MPI_INFO_NULL;
|
||||
// access_id = H5Pcreate(H5P_FILE_ACCESS);
|
||||
// hid_t ret=H5Pset_fapl_mpio(access_id,comm->getMPI(),info);
|
||||
// xfer_plist = H5Pcreate(H5P_DATASET_XFER);
|
||||
// H5Pset_dxpl_mpio(xfer_plist,H5FD_MPIO_COLLECTIVE);
|
||||
// use_pdf=true;
|
||||
// use_collective=false; // everynode writes something
|
||||
#if defined(ENABLE_PHDF5)
|
||||
// enable parallel I/O
|
||||
MPI_Info info=MPI_INFO_NULL;
|
||||
access_id = H5Pcreate(H5P_FILE_ACCESS);
|
||||
#if H5_VERSION_GE(1,10,0)
|
||||
H5Pset_all_coll_metadata_ops(access_id,true);
|
||||
H5Pset_coll_metadata_write(access_id,true);
|
||||
#endif
|
||||
H5Pset_fapl_mpio(access_id,comm->getMPI(),info);
|
||||
xfer_plist = H5Pcreate(H5P_DATASET_XFER);
|
||||
// enable parallel collective I/O
|
||||
H5Pset_dxpl_mpio(xfer_plist,H5FD_MPIO_COLLECTIVE);
|
||||
use_phdf5=true;
|
||||
#else
|
||||
use_phdf5=false;
|
||||
#endif
|
||||
}
|
||||
Mode.set(IS_PARALLEL,use_pdf);
|
||||
//true, if this task does not need to participate in I/O
|
||||
if(use_collective)
|
||||
Mode.set(NOIO,comm->rank());
|
||||
Mode.set(IS_PARALLEL,use_phdf5);
|
||||
Mode.set(IS_MASTER,!comm->rank());
|
||||
if(request_pio&&!use_phdf5)
|
||||
Mode.set(NOIO,comm->rank()); // master only
|
||||
else
|
||||
Mode.set(NOIO,false);
|
||||
Mode.set(NOIO,false); // pio or all.
|
||||
}
|
||||
else
|
||||
{
|
||||
Mode.set(IS_PARALLEL,false);
|
||||
Mode.set(IS_MASTER,true);
|
||||
Mode.set(NOIO,false);
|
||||
}
|
||||
}
|
||||
|
||||
bool hdf_archive::create(const std::string& fname, unsigned flags)
|
||||
{
|
||||
//not I/O node, do nothing
|
||||
if(Mode[NOIO]) return true;
|
||||
if(!(Mode[IS_PARALLEL]||Mode[IS_MASTER])) std::runtime_error("Only create file in parallel or by master but not every rank!");
|
||||
close();
|
||||
file_id = H5Fcreate(fname.c_str(),H5F_ACC_TRUNC,H5P_DEFAULT,access_id);
|
||||
return file_id != is_closed;
|
||||
|
@ -91,8 +99,7 @@ bool hdf_archive::create(const std::string& fname, unsigned flags)
|
|||
|
||||
bool hdf_archive::open(const std::string& fname,unsigned flags)
|
||||
{
|
||||
if(Mode[NOIO])
|
||||
return true;
|
||||
if(Mode[NOIO]) return true;
|
||||
close();
|
||||
file_id = H5Fopen(fname.c_str(),flags,access_id);
|
||||
return file_id != is_closed;
|
||||
|
@ -100,10 +107,8 @@ bool hdf_archive::open(const std::string& fname,unsigned flags)
|
|||
|
||||
bool hdf_archive::is_group(const std::string& aname)
|
||||
{
|
||||
if(Mode[NOIO])
|
||||
return true;
|
||||
if(file_id==is_closed)
|
||||
return false;
|
||||
if(Mode[NOIO]) return true;
|
||||
if(file_id==is_closed) return false;
|
||||
hid_t p=group_id.empty()? file_id:group_id.top();
|
||||
p=(aname[0]=='/')?file_id:p;
|
||||
hid_t g=H5Gopen(p,aname.c_str());
|
||||
|
@ -115,8 +120,7 @@ bool hdf_archive::is_group(const std::string& aname)
|
|||
|
||||
hid_t hdf_archive::push(const std::string& gname, bool createit)
|
||||
{
|
||||
if(Mode[NOIO]||file_id==is_closed)
|
||||
return is_closed;
|
||||
if(Mode[NOIO]||file_id==is_closed) return is_closed;
|
||||
hid_t p=group_id.empty()? file_id:group_id.top();
|
||||
hid_t g=H5Gopen(p,gname.c_str());
|
||||
if(g<0 && createit)
|
||||
|
|
|
@ -34,14 +34,16 @@ class Communicate;
|
|||
|
||||
namespace qmcplusplus
|
||||
{
|
||||
|
||||
/** class to handle hdf file
|
||||
*/
|
||||
struct hdf_archive
|
||||
{
|
||||
enum {IS_PARALLEL=0, NOIO};
|
||||
enum {IS_PARALLEL=0, IS_MASTER, NOIO};
|
||||
static const hid_t is_closed=-1;
|
||||
/** bitset of the io mode
|
||||
* Mode[IS_PARALLEL] : true, if collective
|
||||
* Mode[IS_PARALLEL] : true, if parallel
|
||||
* Mode[IS_MASTER] : true, if the node is master
|
||||
* Mode[NOIO] : true, if I/O is not performed
|
||||
*/
|
||||
std::bitset<4> Mode;
|
||||
|
@ -59,19 +61,22 @@ struct hdf_archive
|
|||
std::stack<hid_t> group_id;
|
||||
/** constructor
|
||||
* @param c communicator
|
||||
* @param use_collective turn on/off collective
|
||||
* @param request_pio turns on parallel I/O,
|
||||
* if ture and PHDF5 is available, hdf_archive is in parallel collective IO mode
|
||||
* if ture and PHDF5 is not available, hdf_archive is in master-only IO mode
|
||||
* if false, hdf_archive is in independent IO mode
|
||||
*/
|
||||
hdf_archive(Communicate* c=0, bool use_collective=false);
|
||||
hdf_archive(Communicate* c=0, bool request_pio=false);
|
||||
///destructor
|
||||
~hdf_archive();
|
||||
|
||||
///set the access property
|
||||
void set_access_plist(bool use_collective, Communicate* comm);
|
||||
void set_access_plist(bool request_pio, Communicate* comm);
|
||||
|
||||
///return true if collective i/o
|
||||
inline bool is_collective() const
|
||||
///return true if parallel i/o
|
||||
inline bool is_parallel() const
|
||||
{
|
||||
return Mode[0];
|
||||
return Mode[IS_PARALLEL];
|
||||
}
|
||||
|
||||
/** create a file
|
||||
|
@ -128,8 +133,8 @@ struct hdf_archive
|
|||
|
||||
template<typename T> bool write(T& data, const std::string& aname)
|
||||
{
|
||||
if(Mode[NOIO])
|
||||
return true;
|
||||
if(Mode[NOIO]) return true;
|
||||
if(!(Mode[IS_PARALLEL]||Mode[IS_MASTER])) std::runtime_error("Only write data in parallel or by master but not every rank!");
|
||||
hid_t p=group_id.empty()? file_id:group_id.top();
|
||||
h5data_proxy<T> e(data);
|
||||
return e.write(p,aname,xfer_plist);
|
||||
|
@ -137,8 +142,7 @@ struct hdf_archive
|
|||
|
||||
template<typename T> bool read(T& data, const std::string& aname)
|
||||
{
|
||||
if(Mode[NOIO])
|
||||
return true;
|
||||
if(Mode[NOIO]) return true;
|
||||
hid_t p=group_id.empty()? file_id:group_id.top();
|
||||
h5data_proxy<T> e(data);
|
||||
return e.read(p,aname,xfer_plist);
|
||||
|
@ -146,8 +150,7 @@ struct hdf_archive
|
|||
|
||||
inline void unlink(const std::string& aname)
|
||||
{
|
||||
if(Mode[NOIO])
|
||||
return;
|
||||
if(Mode[NOIO]) return;
|
||||
hid_t p=group_id.empty()? file_id:group_id.top();
|
||||
herr_t status=H5Gunlink(p,aname.c_str());
|
||||
}
|
||||
|
|
|
@ -115,6 +115,7 @@ bool h5d_read(hid_t grp, const std::string& aname, hsize_t ndims,
|
|||
return ret != -1;
|
||||
}
|
||||
|
||||
|
||||
template<typename T>
|
||||
inline bool h5d_write(hid_t grp, const std::string& aname, hsize_t ndims,
|
||||
const hsize_t* gcounts, const hsize_t* counts, const hsize_t* offsets,
|
||||
|
@ -124,22 +125,35 @@ inline bool h5d_write(hid_t grp, const std::string& aname, hsize_t ndims,
|
|||
return true;
|
||||
hid_t h5d_type_id=get_h5_datatype(*first);
|
||||
hid_t h1 = H5Dopen(grp, aname.c_str());
|
||||
hid_t filespace, memspace;
|
||||
herr_t ret=-1;
|
||||
if(h1<0) //missing create one
|
||||
{
|
||||
hid_t dataspace=H5Screate_simple(ndims,gcounts,NULL);
|
||||
hid_t dataset=H5Dcreate(grp, aname.c_str(),h5d_type_id, dataspace, H5P_DEFAULT);
|
||||
|
||||
hid_t filespace=H5Dget_space(dataspace);
|
||||
hid_t filespace=H5Dget_space(dataset);
|
||||
ret=H5Sselect_hyperslab(filespace,H5S_SELECT_SET,offsets,NULL,counts,NULL);
|
||||
|
||||
hid_t memspace=H5Screate_simple(ndims,counts,NULL);
|
||||
ret=H5Dwrite(dataset,h5d_type_id,memspace,filespace,xfer_plist,first);
|
||||
|
||||
H5Dclose(memspace);
|
||||
H5Sclose(filespace);
|
||||
H5Dclose(dataset);
|
||||
H5Sclose(dataspace);
|
||||
}
|
||||
else
|
||||
{
|
||||
filespace=H5Dget_space(h1);
|
||||
ret=H5Sselect_hyperslab(filespace,H5S_SELECT_SET,offsets,NULL,counts,NULL);
|
||||
|
||||
memspace=H5Screate_simple(ndims,counts,NULL);
|
||||
ret = H5Dwrite(h1, h5d_type_id, memspace, filespace, xfer_plist, first);
|
||||
|
||||
H5Sclose(filespace);
|
||||
H5Dclose(memspace);
|
||||
}
|
||||
H5Dclose(h1);
|
||||
return ret != -1;
|
||||
}
|
||||
|
|
|
@ -4,9 +4,9 @@
|
|||
//
|
||||
// Copyright (c) 2016 Jeongnim Kim and QMCPACK developers.
|
||||
//
|
||||
// File developed by: Jeremy McMinnis, jmcminis@gmail.com, University of Illinois at Urbana-Champaign
|
||||
// File developed by: Jeremy McMinnis, jmcminis@gmail.com, University of Illinois at Urbana-Champaign
|
||||
//
|
||||
// File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
|
||||
// File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
@ -40,7 +40,7 @@ struct hyperslab_proxy: public container_proxy<CT>
|
|||
///1D
|
||||
hyperslab_proxy(CT& a): container_proxy<CT>(a), slab_rank(a.slab_rank),
|
||||
slab_dims(a.slab_dims), slab_offset(a.slab_offset)
|
||||
{
|
||||
{
|
||||
slab_dims_local=slab_dims;
|
||||
use_slab=false;
|
||||
}
|
||||
|
@ -105,7 +105,7 @@ struct h5data_proxy<hyperslab_proxy<CT,MAXDIM> >
|
|||
{
|
||||
return h5d_read(grp,aname.c_str(),
|
||||
ref_.slab_rank,
|
||||
ref_.slab_dims.data(),
|
||||
ref_.slab_dims.data(),
|
||||
ref_.slab_dims_local.data(),
|
||||
ref_.slab_offset.data(),
|
||||
ref_.data(),xfer_plist);
|
||||
|
@ -122,18 +122,18 @@ struct h5data_proxy<hyperslab_proxy<CT,MAXDIM> >
|
|||
}
|
||||
inline bool write(hid_t grp, const std::string& aname, hid_t xfer_plist=H5P_DEFAULT)
|
||||
{
|
||||
//if(ref_.use_slab)
|
||||
//{
|
||||
// std::cout << "Everyone writes a part " << ref_.slab_dims_local[0] << std::endl;
|
||||
// return h5d_write(grp,aname.c_str(),
|
||||
// ref_.slab_rank,
|
||||
// ref_.slab_dims.data(),
|
||||
// ref_.slab_dims_local.data(),
|
||||
// ref_.slab_offset.data(),
|
||||
// ref_.data(),xfer_plist);
|
||||
//}
|
||||
//else
|
||||
if(ref_.use_slab)
|
||||
{
|
||||
return h5d_write(grp,aname.c_str(),
|
||||
ref_.slab_rank,
|
||||
ref_.slab_dims.data(),
|
||||
ref_.slab_dims_local.data(),
|
||||
ref_.slab_offset.data(),
|
||||
ref_.data(),xfer_plist);
|
||||
}
|
||||
else{
|
||||
return h5d_write(grp,aname.c_str(),ref_.slab_rank,ref_.slab_dims.data(),ref_.data(),xfer_plist);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
|
@ -26,6 +26,7 @@
|
|||
#include <Utilities/Timer.h>
|
||||
#include <miniapps/common.hpp>
|
||||
#include <getopt.h>
|
||||
#include <mpi/collectives.h>
|
||||
|
||||
using namespace std;
|
||||
using namespace qmcplusplus;
|
||||
|
@ -49,12 +50,13 @@ int main(int argc, char** argv)
|
|||
Communicate* myComm=OHMMS::Controller;
|
||||
myComm->setName("restart");
|
||||
|
||||
typedef QMCTraits::RealType RealType;
|
||||
typedef ParticleSet::ParticlePos_t ParticlePos_t;
|
||||
typedef ParticleSet::ParticleLayout_t LatticeType;
|
||||
typedef ParticleSet::TensorType TensorType;
|
||||
typedef ParticleSet::PosType PosType;
|
||||
typedef RandomGenerator_t::uint_type uint_type;
|
||||
typedef QMCTraits::RealType RealType;
|
||||
typedef ParticleSet::ParticlePos_t ParticlePos_t;
|
||||
typedef ParticleSet::ParticleLayout_t LatticeType;
|
||||
typedef ParticleSet::TensorType TensorType;
|
||||
typedef ParticleSet::PosType PosType;
|
||||
typedef RandomGenerator_t::uint_type uint_type;
|
||||
typedef MCWalkerConfiguration::Walker_t Walker_t;
|
||||
|
||||
//use the global generator
|
||||
|
||||
|
@ -100,9 +102,8 @@ int main(int argc, char** argv)
|
|||
// set the number of walkers equal to the threads.
|
||||
if(!AverageWalkersPerNode) AverageWalkersPerNode=NumThreads;
|
||||
//set nwtot, to be random
|
||||
nwtot=AverageWalkersPerNode;
|
||||
nwtot=std::abs(AverageWalkersPerNode+myComm->rank()%5-2);
|
||||
FairDivideLow(nwtot,NumThreads,wPerNode);
|
||||
wPerNode.resize(NumThreads+1,0);
|
||||
|
||||
//Random.init(0,1,iseed);
|
||||
Tensor<int,3> tmat(na,0,0,0,nb,0,0,0,nc);
|
||||
|
@ -119,22 +120,23 @@ int main(int argc, char** argv)
|
|||
|
||||
RandomNumberControl::make_seeds();
|
||||
std::vector<RandomGenerator_t> myRNG(NumThreads);
|
||||
std::vector<uint_type> mt(Random.state_size(),0);
|
||||
std::vector<MCWalkerConfiguration> elecs(NumThreads);
|
||||
|
||||
ParticleSet ions;
|
||||
OHMMS_PRECISION scale=1.0;
|
||||
tile_graphite(ions,tmat,scale);
|
||||
|
||||
#pragma omp parallel reduction(+:t0)
|
||||
{
|
||||
int ip=omp_get_thread_num();
|
||||
|
||||
ParticleSet ions;
|
||||
MCWalkerConfiguration& els=elecs[ip];
|
||||
OHMMS_PRECISION scale=1.0;
|
||||
|
||||
//create generator within the thread
|
||||
myRNG[ip]=*RandomNumberControl::Children[ip];
|
||||
RandomGenerator_t& random_th=myRNG[ip];
|
||||
|
||||
tile_graphite(ions,tmat,scale);
|
||||
|
||||
const int nions=ions.getTotalNum();
|
||||
const int nels=4*nions;
|
||||
const int nels3=3*nels;
|
||||
|
@ -152,18 +154,33 @@ int main(int argc, char** argv)
|
|||
els.RSoA=els.R;
|
||||
}
|
||||
|
||||
if(!ip) elecs[0].createWalkers(nwtot);
|
||||
#pragma omp barrier
|
||||
|
||||
for(MCWalkerConfiguration::iterator wi=elecs[0].begin()+wPerNode[ip]; wi!=elecs[0].begin()+wPerNode[ip+1]; wi++)
|
||||
els.saveWalker(**wi);
|
||||
|
||||
// save random seeds and electron configurations.
|
||||
*RandomNumberControl::Children[ip]=myRNG[ip];
|
||||
//MCWalkerConfiguration els_save(els);
|
||||
|
||||
} //end of omp parallel
|
||||
Random.save(mt);
|
||||
|
||||
elecs[0].createWalkers(nwtot);
|
||||
setWalkerOffsets(elecs[0], myComm);
|
||||
|
||||
//storage variables for timers
|
||||
double h5write = 0.0, h5read = 0.0; //random seed R/W speeds
|
||||
double walkerWrite = 0.0, walkerRead =0.0; //walker R/W speeds
|
||||
Timer h5clock; //timer for the program
|
||||
|
||||
// dump random seeds
|
||||
myComm->barrier();
|
||||
h5clock.restart(); //start timer
|
||||
RandomNumberControl::write("restart",myComm);
|
||||
myComm->barrier();
|
||||
h5write += h5clock.elapsed(); //store timer
|
||||
|
||||
// flush random seeds to zero
|
||||
#pragma omp parallel
|
||||
{
|
||||
|
@ -172,9 +189,15 @@ int main(int argc, char** argv)
|
|||
std::vector<uint_type> vt(random_th.state_size(),0);
|
||||
random_th.load(vt);
|
||||
}
|
||||
std::vector<uint_type> mt_temp(Random.state_size(),0);
|
||||
Random.load(mt_temp);
|
||||
|
||||
// load random seeds
|
||||
myComm->barrier();
|
||||
h5clock.restart(); //start timer
|
||||
RandomNumberControl::read("restart",myComm);
|
||||
myComm->barrier();
|
||||
h5read += h5clock.elapsed(); //store timer
|
||||
|
||||
// validate random seeds
|
||||
int mismatch_count=0;
|
||||
|
@ -189,6 +212,9 @@ int main(int argc, char** argv)
|
|||
for(int i=0; i<random_th.state_size(); i++)
|
||||
if(vt_orig[i]!=vt_load[i]) mismatch_count++;
|
||||
}
|
||||
Random.save(mt_temp);
|
||||
for(int i=0; i<Random.state_size(); i++)
|
||||
if(mt_temp[i]!=mt[i]) mismatch_count++;
|
||||
|
||||
myComm->allreduce(mismatch_count);
|
||||
|
||||
|
@ -196,16 +222,27 @@ int main(int argc, char** argv)
|
|||
{
|
||||
if(mismatch_count!=0)
|
||||
std::cout << "Fail: random seeds mismatch between write and read!\n"
|
||||
<< "state_size= " << myRNG[0].state_size() << " mismatch_cout=" << mismatch_count << std::endl;
|
||||
<< " state_size= " << myRNG[0].state_size() << " mismatch_cout=" << mismatch_count << std::endl;
|
||||
else
|
||||
std::cout << "Pass: random seeds match exactly between write and read!\n";
|
||||
}
|
||||
|
||||
// dump electron coordinates.
|
||||
HDFWalkerOutput wOut(elecs[0],"restart",myComm);
|
||||
myComm->barrier();
|
||||
h5clock.restart(); //start timer
|
||||
wOut.dump(elecs[0],1);
|
||||
myComm->barrier();
|
||||
walkerWrite += h5clock.elapsed(); //store timer
|
||||
if(!myComm->rank()) std::cout << "Walkers are dumped!\n";
|
||||
|
||||
// save walkers before destroying them
|
||||
std::vector<Walker_t> saved_walkers;
|
||||
for(int wi=0; wi<elecs[0].getActiveWalkers(); wi++)
|
||||
saved_walkers.push_back(*elecs[0][wi]);
|
||||
elecs[0].destroyWalkers(elecs[0].begin(),elecs[0].end());
|
||||
|
||||
// load walkers
|
||||
const char *restart_input = \
|
||||
"<tmp> \
|
||||
<mcwalkerset fileroot=\"restart\" node=\"-1\" version=\"3 0\" collected=\"yes\"/> \
|
||||
|
@ -219,8 +256,68 @@ int main(int argc, char** argv)
|
|||
|
||||
HDFVersion in_version(0,4);
|
||||
HDFWalkerInput_0_4 wIn(elecs[0],myComm,in_version);
|
||||
myComm->barrier();
|
||||
h5clock.restart(); //start timer
|
||||
wIn.put(restart_leaf);
|
||||
|
||||
myComm->barrier();
|
||||
walkerRead += h5clock.elapsed(); //store time spent
|
||||
|
||||
if(saved_walkers.size()!=elecs[0].getActiveWalkers())
|
||||
std::cout << "Fail: Rank " << myComm->rank() << " had " << saved_walkers.size()
|
||||
<< " walkers but loaded only " << elecs[0].getActiveWalkers() << " walkers from file!" << std::endl;
|
||||
|
||||
mismatch_count=0;
|
||||
for(int wi=0; wi<saved_walkers.size(); wi++)
|
||||
{
|
||||
saved_walkers[wi].R=saved_walkers[wi].R-elecs[0][wi]->R;
|
||||
if(Dot(saved_walkers[wi].R,saved_walkers[wi].R)>std::numeric_limits<RealType>::epsilon()) mismatch_count++;
|
||||
}
|
||||
myComm->allreduce(mismatch_count);
|
||||
|
||||
if(!myComm->rank())
|
||||
{
|
||||
if(mismatch_count!=0)
|
||||
std::cout << "Fail: electron coordinates mismatch between write and read!\n"
|
||||
<< " mismatch_cout=" << mismatch_count << std::endl;
|
||||
else
|
||||
std::cout << "Pass: electron coordinates match exactly between write and read!\n";
|
||||
}
|
||||
|
||||
//print out hdf5 R/W times
|
||||
TinyVector<double,4> timers(h5read, h5write, walkerRead, walkerWrite);
|
||||
mpi::reduce(*myComm, timers);
|
||||
h5read = timers[0]/myComm->size();
|
||||
h5write = timers[1]/myComm->size();
|
||||
walkerRead = timers[2]/myComm->size();
|
||||
walkerWrite = timers[3]/myComm->size();
|
||||
if(myComm->rank() == 0)
|
||||
{
|
||||
cout << "\nTotal time of writing random seeds to HDF5 file: " << setprecision(2) << h5write << "\n";
|
||||
cout << "\nTotal time of reading random seeds in HDF5 file: " << setprecision(2) << h5read << "\n";
|
||||
cout << "\nTotal time of writing walkers to HDF5 file: " << setprecision(2) << walkerWrite << "\n";
|
||||
cout << "\nTotal time of reading walkers in HDF5 file: " << setprecision(2) << walkerRead << "\n";
|
||||
}
|
||||
|
||||
if(myComm->size()>1)
|
||||
{
|
||||
Communicate* subComm = new Communicate(*myComm, 2);
|
||||
subComm->setName("restart2");
|
||||
|
||||
if(subComm->getGroupID() == 0)
|
||||
{
|
||||
elecs[0].destroyWalkers(elecs[0].begin(),elecs[0].end());
|
||||
HDFWalkerInput_0_4 subwIn(elecs[0],subComm,in_version);
|
||||
subwIn.put(restart_leaf);
|
||||
subComm->barrier();
|
||||
if(!subComm->rank()) std::cout << "Walkers are loaded again by the subgroup!\n";
|
||||
setWalkerOffsets(elecs[0], subComm);
|
||||
HDFWalkerOutput subwOut(elecs[0],"XXXX",subComm);
|
||||
subwOut.dump(elecs[0],1);
|
||||
if(!subComm->rank()) std::cout << "Walkers are dumped again by the subgroup!\n";
|
||||
}
|
||||
delete subComm;
|
||||
}
|
||||
|
||||
OHMMS::Controller->finalize();
|
||||
|
||||
return 0;
|
||||
|
|
Loading…
Reference in New Issue