From 4b6336367998b5706a33db473cfc1576bc14cf0b Mon Sep 17 00:00:00 2001 From: Vladimir Mironov Date: Sun, 4 Feb 2018 08:55:52 -0500 Subject: [PATCH] BigMPI compatibility BigMPI support for very large jobs See https://github.com/jeffhammond/BigMPI for details --- interface/makefile | 6 +++- make.inc | 16 ++++++++--- qureg/qureg_apply1qubitgate.cpp | 2 +- qureg/qureg_permute.cpp | 5 ++++ qureg/qureg_utils.cpp | 13 +++++++++ tests/makefile | 16 ++++++----- util/utils.hpp | 50 ++++++++++++++++++++++++++------- 7 files changed, 85 insertions(+), 23 deletions(-) diff --git a/interface/makefile b/interface/makefile index 525199a..40632f4 100644 --- a/interface/makefile +++ b/interface/makefile @@ -18,6 +18,10 @@ include ../make.inc LDFLAGS = -L${MKLROOT}/lib/intel64 -lmkl_cdft_core -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_ilp64 -lpthread +ifdef bigmpi + LDFLAGS := ${BIGMPI_LIB} ${LDFLAGS} +endif + SRC = $(wildcard *.cpp) OBJS = $(addsuffix .o, $(basename $(SRC))) EXEC = qasm_interface.exe @@ -27,7 +31,7 @@ EXEC = qasm_interface.exe all: $(EXEC) $(EXEC): ${OBJS} makefile ../qureg/qHiPSTER.a - $(CC) ${CFLAGS} ${LDFLAGS} ${OBJS} ../qureg/qHiPSTER.a -o $(EXEC) + $(CC) ${CFLAGS} ${OBJS} ../qureg/qHiPSTER.a ${LDFLAGS} -o $(EXEC) clean: rm -f *.o *.s *.d *~ $(EXEC) diff --git a/make.inc b/make.inc index 730839c..ef5b919 100644 --- a/make.inc +++ b/make.inc @@ -14,26 +14,34 @@ ## limitations under the License. ##------------------------------------------------------------------------------ +bigmpi = 1 + HOST = $(shell hostname) CC = g++ CC = CC -mkl -no-ipo CC = mpicxx -CC = mpiicpc +CC = mpiicpc -OPTS = -g -OPTS = -O3 +OPTS = -g -O3 VECS = -qopt-report-phase=vec -qopt-report=2 DEFS = -DUSE_MM_MALLOC -DSTANDALONE -DOPENQU_HAVE_MPI -DMKL_ILP64 +BIGMPI_LIB = /opt/bigmpi/lib/libbigmpi.a +BIGMPI_INC = -I/opt/bigmpi/include + CFLAGS = ${OPTS} ${DEFS} \ -Wall -std=c++11 -qopenmp #-restrict \ # -fasm-blocks -CFLAGS += ${OPTFLAGS} +CFLAGS += ${OPTFLAGS} INC = -I./ -I../ +ifdef bigmpi + DEFS := ${DEFS} -DBIGMPI + INC := ${INC} ${BIGMPI_INC} +endif ########### Modified for the ccefw-desktop.jf.intel.com environment ############ ifeq ($(HOST),ccefw-desktop) diff --git a/qureg/qureg_apply1qubitgate.cpp b/qureg/qureg_apply1qubitgate.cpp index 566d0bf..7b7e8d9 100644 --- a/qureg/qureg_apply1qubitgate.cpp +++ b/qureg/qureg_apply1qubitgate.cpp @@ -106,7 +106,7 @@ double QbitRegister::HP_Distrpair(unsigned pos, TM2x2 const&m) Type *tmpstate = tmpspace(); std::size_t lcl_size_half = localSize() / 2L; - assert(lcl_size_half <= std::numeric_limits::max()); + assert(lcl_size_half <= std::numeric_limits::max()); if (lcl_chunk > lcl_size_half) lcl_chunk = lcl_size_half; diff --git a/qureg/qureg_permute.cpp b/qureg/qureg_permute.cpp index 6eb6088..4899bd2 100644 --- a/qureg/qureg_permute.cpp +++ b/qureg/qureg_permute.cpp @@ -42,8 +42,13 @@ void QbitRegister::Permute(std::vector permutation_new_vec) // Dummy multi-node permutation code std::vector glb_state(globalSize(), 0); +#ifdef BIGMPI + MPIX_Allgather_x(&(state[0]), localSize(), MPI_DOUBLE_COMPLEX, &(glb_state[0]), localSize(), + MPI_DOUBLE_COMPLEX, comm); +#else MPI_Allgather(&(state[0]), localSize(), MPI_DOUBLE_COMPLEX, &(glb_state[0]), localSize(), MPI_DOUBLE_COMPLEX, comm); +#endif //BIGMPI for (std::size_t i = 0; i < glb_state.size(); i++) { std::size_t glbind = permutation_old.bin2dec(permutation_new.perm2lin(permutation_old.lin2perm(i))); diff --git a/qureg/qureg_utils.cpp b/qureg/qureg_utils.cpp index 48e8780..1dafe92 100644 --- a/qureg/qureg_utils.cpp +++ b/qureg/qureg_utils.cpp @@ -366,9 +366,17 @@ void QbitRegister::Print(std::string x, std::vector qbits) #ifdef OPENQU_HAVE_MPI for (std::size_t i = 1; i < nprocs; i++) { std::size_t len; +#ifdef BIGMPI + MPIX_Recv_x(&len, 1, MPI_LONG, i, 1000 + i, comm, MPI_STATUS_IGNORE); +#else MPI_Recv(&len, 1, MPI_LONG, i, 1000 + i, comm, MPI_STATUS_IGNORE); +#endif //BIGMPI s.resize(len); +#ifdef BIGMPI + MPIX_Recv_x((void *)(s.c_str()), len, MPI_CHAR, i, i, comm, MPI_STATUS_IGNORE); +#else MPI_Recv((void *)(s.c_str()), len, MPI_CHAR, i, i, comm, MPI_STATUS_IGNORE); +#endif //BIGMPI printf("%s", s.c_str()); } #endif @@ -377,8 +385,13 @@ void QbitRegister::Print(std::string x, std::vector qbits) #ifdef OPENQU_HAVE_MPI std::string s = printvec(state, localSize(), nqbits, pcum, permutation); std::size_t len = s.length() + 1; +#ifdef BIGMPI + MPIX_Send_x(&len, 1, MPI_LONG, 0, 1000 + rank, comm); + MPIX_Send_x(const_cast(s.c_str()), len, MPI_CHAR, 0, rank, comm); +#else MPI_Send(&len, 1, MPI_LONG, 0, 1000 + rank, comm); MPI_Send(const_cast(s.c_str()), len, MPI_CHAR, 0, rank, comm); +#endif //BIGMPI #endif } diff --git a/tests/makefile b/tests/makefile index ea5b6e1..c529e1d 100644 --- a/tests/makefile +++ b/tests/makefile @@ -17,12 +17,14 @@ include ../make.inc LDFLAGS = -L${MKLROOT}/lib/intel64 -lmkl_cdft_core -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_ilp64 -lpthread -LDFLAGS = -L$(MKLROOT)lib/intel64 -lmkl_cdft_core -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_lp64 -lpthread \ - ifdef memcheck LDFLAGS := ${LDFLAGS} /home/msmelyan/valgrind-3.11.0/installed/lib/valgrind/libmpiwrap-amd64-linux.so endif +ifdef bigmpi + LDFLAGS := ${BIGMPI_LIB} ${LDFLAGS} +endif + EXECS := benchgates.exe\ testgates.exe\ qft_test.exe\ @@ -32,15 +34,15 @@ EXECS := benchgates.exe\ all: ${EXECS} benchgates.exe: benchgates.o makefile ../qureg/qHiPSTER.a - $(CC) -o benchgates.exe ${CFLAGS} benchgates.o ${LDFLAGS} ../qureg/qHiPSTER.a + $(CC) -o benchgates.exe ${CFLAGS} benchgates.o ../qureg/qHiPSTER.a ${LDFLAGS} testgates.exe: testgates.o makefile ../qureg/qHiPSTER.a - $(CC) -o testgates.exe ${CFLAGS} testgates.o ${LDFLAGS} ../qureg/qHiPSTER.a + $(CC) -o testgates.exe ${CFLAGS} testgates.o ../qureg/qHiPSTER.a ${LDFLAGS} qft_test.exe: qft_test.o makefile ../qureg/qHiPSTER.a - $(CC) -o qft_test.exe ${CFLAGS} qft_test.o ${LDFLAGS} ../qureg/qHiPSTER.a + $(CC) -o qft_test.exe ${CFLAGS} qft_test.o ../qureg/qHiPSTER.a ${LDFLAGS} expect_value_test.exe: expect_value_test.o makefile ../qureg/qHiPSTER.a - $(CC) -o expect_value_test.exe ${CFLAGS} expect_value_test.o ${LDFLAGS} ../qureg/qHiPSTER.a + $(CC) -o expect_value_test.exe ${CFLAGS} expect_value_test.o ../qureg/qHiPSTER.a ${LDFLAGS} noisy_circuit_test.exe: noisy_circuit_test.o makefile ../qureg/qHiPSTER.a - $(CC) -o noisy_circuit_test.exe ${CFLAGS} noisy_circuit_test.o ${LDFLAGS} ../qureg/qHiPSTER.a + $(CC) -o noisy_circuit_test.exe ${CFLAGS} noisy_circuit_test.o ../qureg/qHiPSTER.a ${LDFLAGS} %.o : %.cpp makefile $(CC) ${CFLAGS} ${INC} ${LDFLAGS} -c $< -o $@ diff --git a/util/utils.hpp b/util/utils.hpp index 2841572..b46c25f 100644 --- a/util/utils.hpp +++ b/util/utils.hpp @@ -26,6 +26,11 @@ #include #include +#ifdef BIGMPI +#include "mpi.hpp" +#include "bigmpi.h" +#endif //BIGMPI + // defines #define DO_PRAGMA(x) _Pragma(#x) @@ -65,38 +70,63 @@ int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, static int MPI_Allreduce_x(float *sendbuf, float *recvbuf, MPI_Op op, MPI_Comm comm) { +#ifdef BIGMPI + return MPIX_Allreduce_x((void*)sendbuf, (void *)recvbuf, 1, + MPI_FLOAT, op, comm); +#else return MPI_Allreduce((void*)sendbuf, (void *)recvbuf, 1, MPI_FLOAT, op, comm); +#endif //BIGMPI } static int MPI_Allreduce_x(double *sendbuf, double *recvbuf, MPI_Op op, MPI_Comm comm) { +#ifdef BIGMPI + return MPIX_Allreduce_x((void*)sendbuf, (void *)recvbuf, 1, + MPI_DOUBLE, op, comm); +#else return MPI_Allreduce((void*)sendbuf, (void *)recvbuf, 1, MPI_DOUBLE, op, comm); +#endif //BIGMPI } static -int MPI_Sendrecv_x(ComplexSP *sendbuf, int sendcount, - int dest, int sendtag, - ComplexSP *recvbuf, int recvcount, - int source, int recvtag, +int MPI_Sendrecv_x(ComplexSP *sendbuf, size_t sendcount, + size_t dest, size_t sendtag, + ComplexSP *recvbuf, size_t recvcount, + size_t source, size_t recvtag, MPI_Comm comm, MPI_Status *status) +#ifdef BIGMPI { return - MPI_Sendrecv((void *)sendbuf, sendcount, MPI_COMPLEX, dest, sendtag, + MPIX_Sendrecv_x((void *)sendbuf, sendcount, MPI_COMPLEX, dest, sendtag, (void *)recvbuf, recvcount, MPI_COMPLEX, source, recvtag, comm, status); } +#else +{ + return + MPI_Sendrecv((void *)sendbuf, sendcount, MPI_COMPLEX, dest, sendtag, + (void *)recvbuf, recvcount, MPI_COMPLEX, source, recvtag, comm, status); +} +#endif //BIGMPI static -int MPI_Sendrecv_x(ComplexDP *sendbuf, int sendcount, - int dest, int sendtag, - ComplexDP *recvbuf, int recvcount, - int source, int recvtag, +int MPI_Sendrecv_x(ComplexDP *sendbuf, size_t sendcount, + size_t dest, size_t sendtag, + ComplexDP *recvbuf, size_t recvcount, + size_t source, size_t recvtag, MPI_Comm comm, MPI_Status *status) +#ifdef BIGMPI +{ + return + MPIX_Sendrecv_x((void *)sendbuf, sendcount, MPI_DOUBLE_COMPLEX, dest, sendtag, + (void *)recvbuf, recvcount, MPI_DOUBLE_COMPLEX, source, recvtag, comm, status); +} +#else { return MPI_Sendrecv((void *)sendbuf, sendcount, MPI_DOUBLE_COMPLEX, dest, sendtag, (void *)recvbuf, recvcount, MPI_DOUBLE_COMPLEX, source, recvtag, comm, status); } - +#endif //BIGMPI