Can compile without GPU code. Added missing sync calls.

Modules/wavefunctions_gpu.f90

@@ -33,10 +33,11 @@
      CONTAINS
      !
      SUBROUTINE using_evc(changing)
-#if defined(__CUDA)
+         !
          USE wavefunctions_module, ONLY : evc
          implicit none
          LOGICAL, INTENT(IN) :: changing
+#if defined(__CUDA)
          !
          IF (evc_ood) THEN
              IF (.not. allocated(evc_d)) THEN
@@ -57,10 +58,11 @@
      END SUBROUTINE using_evc
      !
      SUBROUTINE using_evc_d(changing)
-#if defined(__CUDA)
+         !
          USE wavefunctions_module, ONLY : evc
          implicit none
          LOGICAL, INTENT(IN) :: changing
+#if defined(__CUDA)
          !
          IF (.not. allocated(evc)) THEN
              IF (allocated(evc_d)) DEALLOCATE(evc_d)

PW/src/c_bands.f90

@@ -376,7 +376,7 @@ CONTAINS
              END IF
              CALL using_evc_d(.true.) !evc_d = evc
 #else
-             CALL errore( ' diag_bands '. ' Called GPU version which is not available!', 1)
+             CALL errore( ' diag_bands ', ' Called GPU version of c_bands which is not available!', 1)
 #endif
           END IF
           !
@@ -528,6 +528,7 @@ CONTAINS
                                et(1,ik), btype(1,ik), notconv, lrot, dav_iter )
              END IF
           ELSE
+#if defined(__CUDA)
              CALL using_evc_d(.true.) !evc_d = evc
              IF ( use_para_diag ) then
                 !
@@ -550,6 +551,9 @@ CONTAINS
                 et(:,ik) = et_d
                 DEALLOCATE(et_d)
              END IF
+#else
+             CALL errore( ' diag_bands ', ' Called GPU version of c_bands which is not available!', 1)
+#endif
           END IF
           !
           avg_iter = avg_iter + dav_iter

PW/src/exx_band.f90

@@ -84,6 +84,8 @@ MODULE exx_band
     USE control_flags,        ONLY : io_level
     USE buffers,              ONLY : open_buffer, get_buffer, save_buffer
     USE mp_exx,               ONLY : max_ibands, negrp
+
+    USE wavefunctions_module_gpum, ONLY : using_evc
     !
     !
     IMPLICIT NONE
@@ -92,6 +94,8 @@ MODULE exx_band
     INTEGER :: lda, n, ik
     LOGICAL :: exst_mem, exst_file
     !
+    CALL using_evc(.false.)
+    !
     IF (negrp.eq.1) THEN
        !
        ! no change in data structure is necessary
@@ -164,6 +168,7 @@ MODULE exx_band
        ! read evc for the local data structure
        !
        IF ( nks > 1 ) CALL get_buffer(evc, nwordwfc, iunwfc, ik)
+       IF ( nks > 1 ) CALL using_evc(.true.)
        !
        ! transform evc to the EXX data structure
        !

PW/src/g_psi_mod_gpu.f90

@@ -5,7 +5,6 @@
 ! in the root directory of the present distribution,
 ! or http://www.gnu.org/copyleft/gpl.txt .
 !
-#if defined(__CUDA)
 MODULE g_psi_mod_gpum
   !
   ! ... These are the variables needed in g_psi
@@ -14,14 +13,18 @@ MODULE g_psi_mod_gpum
   !
   IMPLICIT NONE
   !
-  REAL(DP), DEVICE, ALLOCATABLE :: &
+  REAL(DP), ALLOCATABLE :: &
     h_diag_d (:,:),&   ! diagonal part of the Hamiltonian
     s_diag_d (:,:)     ! diagonal part of the overlap matrix
 
+#if defined(__CUDA)
+  attributes(device) :: h_diag_d, s_diag_d
+
   LOGICAL :: h_diag_ood = .false.    ! used to flag out of date variables
   LOGICAL :: s_diag_ood = .false.
   LOGICAL :: h_diag_d_ood = .false.
   LOGICAL :: s_diag_d_ood = .false.
+#endif
   !
   CONTAINS
   !
@@ -29,12 +32,14 @@ MODULE g_psi_mod_gpum
       USE g_psi_mod, ONLY : h_diag
       implicit none
       LOGICAL, INTENT(IN) :: writing
+#if defined(__CUDA)
       !
       IF (h_diag_ood) THEN
           h_diag = h_diag_d
           h_diag_ood = .false.
       ENDIF
       IF (writing)    h_diag_d_ood = .true.
+#endif
   END SUBROUTINE using_h_diag
   !
   SUBROUTINE using_h_diag_d(writing)
@@ -42,6 +47,7 @@ MODULE g_psi_mod_gpum
       implicit none
       LOGICAL, INTENT(IN) :: writing
       !
+#if defined(__CUDA)
       IF (.not. allocated(h_diag)) THEN
           IF (allocated(h_diag_d)) DEALLOCATE(h_diag_d)
           h_diag_d_ood = .false.
@@ -56,24 +62,28 @@ MODULE g_psi_mod_gpum
           h_diag_d_ood = .false.
       ENDIF
       IF (writing)    h_diag_ood = .true.
+#endif
   END SUBROUTINE using_h_diag_d
   !
   SUBROUTINE using_s_diag(writing)
       USE g_psi_mod, ONLY : s_diag
       implicit none
       LOGICAL, INTENT(IN) :: writing
+#if defined(__CUDA)
       !
       IF (s_diag_ood) THEN
           s_diag = s_diag_d
           s_diag_ood = .false.
       ENDIF
       IF (writing)    s_diag_d_ood = .true.
+#endif
   END SUBROUTINE using_s_diag
   !
   SUBROUTINE using_s_diag_d(writing)
       USE g_psi_mod, ONLY : s_diag
       implicit none
       LOGICAL, INTENT(IN) :: writing
+#if defined(__CUDA)
       !
       IF (.not. allocated(s_diag)) THEN
           IF (allocated(s_diag_d)) DEALLOCATE(s_diag_d)
@@ -89,7 +99,7 @@ MODULE g_psi_mod_gpum
           s_diag_d_ood = .false.
       ENDIF
       IF (writing)    s_diag_ood = .true.
+#endif
   END SUBROUTINE using_s_diag_d
   !
 END MODULE g_psi_mod_gpum
-#endif

PW/src/stres_us.f90

@@ -161,6 +161,7 @@ SUBROUTINE stres_us( ik, gk, sigmanlc )
        !
        ! ... non diagonal contribution - derivative of the bessel function
        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+       CALL using_evc(.false.)
        ALLOCATE( dvkb( npwx, nkb ) )
        !
        CALL gen_us_dj( ik, dvkb )
@@ -223,6 +224,7 @@ SUBROUTINE stres_us( ik, gk, sigmanlc )
        !
        IF ( lmaxkb == 0 ) GO TO 10
        !
+       CALL using_evc(.false.) ! This is redundant
        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
        DO ipol = 1, 3
           CALL gen_us_dy( ik, xyz(1,ipol), dvkb )
@@ -407,6 +409,7 @@ SUBROUTINE stres_us( ik, gk, sigmanlc )
        !
        CALL gen_us_dj( ik, dvkb )
        !
+       CALL using_evc(.false.)
        DO ibnd = 1, nbnd
           IF (noncolin) THEN
              work2_nc = (0.D0,0.D0)