Testprogs BF for openACC

UMR-CNRM · Apr 2, 2024 · 6dc52c4 · 6dc52c4
1 parent 0fd43b4
commit 6dc52c4
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 32 deletions.
diff --git a/build/with_fcm/arch/arch-pgi_CPU_preGPU_MFflavour.env b/build/with_fcm/arch/arch-pgi_CPU_preGPU_MFflavour.env
@@ -40,6 +40,7 @@ common_opts="$common_opts --deleteDrHook --deleteBudgetDDH" #Not available on GP
 common_opts="$common_opts --addACC_routine_seq" #add 'acc routine' directive
 common_opts="$common_opts --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
 common_opts="$common_opts --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
+common_opts="$common_opts --buildACCTypeHelpers"
 
 #We cannot suppress non column calls for Méso-NH, stack transformation is replaced by allocatables
 #source this file, then: "export PYFT_OPTS_MNH; check_commit_mesonh.sh --prep_code-opts '--pyft_opts_env PYFT_OPTS_MNH' ...."

diff --git a/src/common/aux/modd_io.F90 b/src/common/aux/modd_io.F90
@@ -8,28 +8,7 @@ MODULE MODD_IO
 !
 !Structure describing the characteristics of a file
 TYPE TFILEDATA
-  CHARACTER(LEN=NFILENAMELGTMAX) :: CNAME = '' !Filename
-  CHARACTER(LEN=:),ALLOCATABLE   :: CDIRNAME   !Directory name
-  CHARACTER(LEN=13) :: CTYPE   = "UNKNOWN" !Filetype (PGD, MNH, DES, NML...)
-  CHARACTER(LEN=7)  :: CFORMAT = "UNKNOWN" !Fileformat (NETCDF4, LFI, LFICDF4...)
-  CHARACTER(LEN=7)  :: CMODE   = "UNKNOWN" !Opening mode (read, write...)
   LOGICAL           :: LOPENED = .FALSE.   !Is the file opened
-  INTEGER           :: NOPEN_CURRENT = 0   !Number of times the file is currently opened (several opens without close are allowed)
-  INTEGER           :: NOPEN   = 0         !Number of times the file has been opened (during the current execution)
-  INTEGER           :: NCLOSE  = 0         !Number of times the file has been closed (during the current execution)
-  !
-  INTEGER           :: NMASTER_RANK  = -1      !Rank of the master process (no meaning if LMULTIMASTERS=.T.)
-  INTEGER           :: NMPICOMM      = -1      !MPI communicator used for IO on this file
-  LOGICAL           :: LMASTER       = .FALSE. !True if process is master of the file (process that open/read/write/close)
-  LOGICAL           :: LMULTIMASTERS = .FALSE. !True if several processes may access the file
-  INTEGER           :: NSUBFILES_IOZ = 0       !Number of sub-files (Z-split files based on this file)
-                                               !For example if 2 sub-files and this file is abcd,
-                                               !the 2 sub-files are abcd.Z001 and abcd.Z002
-!  TYPE(TFILE_ELT),DIMENSION(:),ALLOCATABLE :: TFILES_IOZ !Corresponding Z-split files
-  !
-  INTEGER              :: NMODEL = 0              !Model number corresponding to the file (field not always set)
-  INTEGER,DIMENSION(3) :: NMNHVERSION = (/0,0,0/) !MesoNH version used to create the file
-  !
   INTEGER :: NLU = -1 ! logical unit number
 END TYPE TFILEDATA
 ENDMODULE MODD_IO
diff --git a/src/testprogs/ice_adjust/main_ice_adjust.F90 b/src/testprogs/ice_adjust/main_ice_adjust.F90
@@ -10,6 +10,9 @@ PROGRAM MAIN_ICE_ADJUST
 USE MODE_MNH_ZWORK,  ONLY: ZMNH_STACK, IMNH_BLOCK, YMNH_STACK, INUMPIN
 USE OMP_LIB
 USE YOMHOOK, ONLY : LHOOK, DR_HOOK, JPHOOK
+#ifdef _OPENACC
+USE MODD_UTIL_PHYEX_T, ONLY: COPY_PHYEX_T, WIPE_PHYEX_T
+#endif
 
 IMPLICIT NONE
 
@@ -137,7 +140,11 @@ PROGRAM MAIN_ICE_ADJUST
 D0%NKTB = 1
 D0%NKTE = KLEV
 
+#if defined(USE_COLCALL) && defined(_OPENACC)
+ISTSZ = NPROMA * 500 * KLEV
+#else
 ISTSZ = NPROMA * 20 * KLEV
+#endif
 ALLOCATE (PSTACK (ISTSZ, NGPBLKS))
 ZMNH_STACK => PSTACK
 
@@ -152,8 +159,12 @@ PROGRAM MAIN_ICE_ADJUST
 
   TSD = OMP_GET_WTIME ()
 
+#ifdef _OPENACC
+  CALL COPY_PHYEX_T(PHYEX)
+#endif
+
 !$acc data &
-!$acc      & copyin  (D0, PHYEX, &
+!$acc      & copyin  (D0, &
 !$acc      &          ZSIGQSAT, PRHODJ, PEXNREF, PRHODREF, PSIGS, PMFCONV, PPABSM, ZZZ, PCF_MF, PRC_MF, PRI_MF, &
 !$acc      &          ZDUM1, ZDUM2, ZDUM3, ZDUM4, ZDUM5, ZRS, ZICE_CLD_WGT) &
 !$acc      & copy    (PRS, PTHS) &
@@ -176,7 +187,7 @@ PROGRAM MAIN_ICE_ADJUST
 
   D = D0
 
-!$acc parallel loop gang vector private (YLSTACK, IBL, JLON, D) collapse (2)
+!$acc parallel loop gang vector present (PHYEX) private (YLSTACK, IBL, JLON, D) collapse (2)
 
   DO IBL = JBLK1, JBLK2
 
@@ -191,8 +202,14 @@ PROGRAM MAIN_ICE_ADJUST
 
 #ifdef USE_STACK
     !Using cray pointers, AROME mechanism
+#if defined(USE_COLCALL) && defined(_OPENACC)
+    !Due to the collapse (2) directive, each point must have its own stack
+    YLSTACK%L = LOC (PSTACK (1, IBL)) + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8 * (JLON - 1)
+    YLSTACK%U = YLSTACK%L + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8
+#else
     YLSTACK%L = LOC (PSTACK (1, IBL))
     YLSTACK%U = YLSTACK%L + ISTSZ * KIND (PSTACK)
+#endif
 #else
     !Using fortran indexing, Meso-NH mechanism
     YLSTACK%L = 1
@@ -241,6 +258,10 @@ PROGRAM MAIN_ICE_ADJUST
 
 !$acc end data
 
+#ifdef _OPENACC
+  CALL WIPE_PHYEX_T(PHYEX)
+#endif
+
   TED = OMP_GET_WTIME ()
 
   ZTC = ZTC + (TEC - TSC)

diff --git a/src/testprogs/rain_ice/main_rain_ice.F90 b/src/testprogs/rain_ice/main_rain_ice.F90
@@ -10,6 +10,9 @@ PROGRAM MAIN_RAIN_ICE
 USE MODE_MNH_ZWORK,  ONLY: ZMNH_STACK, IMNH_BLOCK, YMNH_STACK, INUMPIN
 USE OMP_LIB
 USE YOMHOOK, ONLY : LHOOK, DR_HOOK, JPHOOK
+#ifdef _OPENACC
+USE MODD_UTIL_PHYEX_T, ONLY: COPY_PHYEX_T, WIPE_PHYEX_T
+#endif
 
 IMPLICIT NONE
 
@@ -132,7 +135,11 @@ PROGRAM MAIN_RAIN_ICE
 D0%NKTB = 1
 D0%NKTE = KLEV
 
+#if defined(USE_COLCALL) && defined(_OPENACC)
+ISTSZ = NPROMA * 2000 * KLEV
+#else
 ISTSZ = NPROMA * 200 * KLEV
+#endif
 ALLOCATE (PSTACK (ISTSZ, NGPBLKS))
 ZMNH_STACK => PSTACK
 
@@ -152,8 +159,12 @@ PROGRAM MAIN_RAIN_ICE
 
   TSD = OMP_GET_WTIME ()
 
+#ifdef _OPENACC
+  CALL COPY_PHYEX_T(PHYEX)
+#endif
+
 !$acc data &
-!$acc      & copyin  (D0, PHYEX, &
+!$acc      & copyin  (D0, &
 !$acc      &          ZTHVREFZIKB, PEXNREF, PDZZ, PRHODJ, PRHODREF, PEXNREF2, PPABSM, PCLDFR, &
 !$acc      &          PTHT, PRT, PSIGS, PSEA, PTOWN) &
 !$acc      & copy    (PCIT, PHLC_HRC, PHLC_HCF, PHLI_HRI, PHLI_HCF, PTHS, PRS) &
@@ -176,7 +187,7 @@ PROGRAM MAIN_RAIN_ICE
 
   D = D0
 
-!$acc parallel loop gang vector private (YLSTACK, IBL, JLON, D) collapse (2)
+!$acc parallel loop gang vector present (PHYEX) private (YLSTACK, IBL, JLON, D) collapse (2)
 
   DO IBL = JBLK1, JBLK2
 
@@ -191,8 +202,14 @@ PROGRAM MAIN_RAIN_ICE
 
 #ifdef USE_STACK
     !Using cray pointers, AROME mechanism
+#if defined(USE_COLCALL) && defined(_OPENACC)
+    !Due to the collapse (2) directive, each point must have its own stack
+    YLSTACK%L = LOC (PSTACK (1, IBL)) + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8 * (JLON - 1)
+    YLSTACK%U = YLSTACK%L + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8
+#else
     YLSTACK%L = LOC (PSTACK (1, IBL))
     YLSTACK%U = YLSTACK%L + ISTSZ * KIND (PSTACK)
+#endif
 #else
     !Using fortran indexing, Meso-NH mechanism
     YLSTACK%L = 1
@@ -243,6 +260,10 @@ PROGRAM MAIN_RAIN_ICE
 
 !$acc end data
 
+#ifdef _OPENACC
+  CALL WIPE_PHYEX_T(PHYEX)
+#endif
+
   TED = OMP_GET_WTIME ()
 
   ZTC = ZTC + (TEC - TSC)

diff --git a/src/testprogs/rain_ice_old/main_rain_ice_old.F90 b/src/testprogs/rain_ice_old/main_rain_ice_old.F90
@@ -11,6 +11,10 @@ PROGRAM MAIN_RAIN_ICE_OLD
 USE OMP_LIB
 USE YOMHOOK, ONLY : LHOOK, DR_HOOK, JPHOOK
 USE MODD_LES,        ONLY: TLES_t
+#ifdef _OPENACC
+USE MODD_UTIL_PHYEX_T, ONLY: COPY_PHYEX_T, WIPE_PHYEX_T
+#endif
+
 
 USE ISO_FORTRAN_ENV, ONLY: OUTPUT_UNIT
 
@@ -195,7 +199,11 @@ PROGRAM MAIN_RAIN_ICE_OLD
 D0%NKTB = 1
 D0%NKTE = KLEV
 
+#if defined(USE_COLCALL) && defined(_OPENACC)
+ISTSZ = NPROMA * 1500 * KLEV
+#else
 ISTSZ = NPROMA * 100 * KLEV
+#endif
 ALLOCATE (PSTACK (ISTSZ, NGPBLKS))
 ZMNH_STACK => PSTACK
 
@@ -216,8 +224,12 @@ PROGRAM MAIN_RAIN_ICE_OLD
 
   TSD = OMP_GET_WTIME ()
 
+#ifdef _OPENACC
+  CALL COPY_PHYEX_T(PHYEX)
+#endif
+
 !$acc data &
-!$acc      & copyin  (D0, PHYEX, TLES, LKOGAN, LMODICEDEP, KKA, KKU, KKL, KSPLITR, PTSTEP, KRR, ISIZEMICRO, &
+!$acc      & copyin  (D0, TLES, LKOGAN, LMODICEDEP, KKA, KKU, KKL, KSPLITR, PTSTEP, KRR, ISIZEMICRO, &
 !$acc      &          LLMICRO, PDZZ, PRHODJ, PRHODREF, PEXNREF, PPABSM, PCLDFR, PICLDFR, PSSIO, PSSIU, &
 !$acc      &          PTHT, PRT, PSIGS, PSEA, PTOWN, PICENU, PKGN_ACON, PKGN_SBGR) &
 !$acc      & copy    (PCIT, PIFR, PTHS, PRS) &
@@ -240,7 +252,7 @@ PROGRAM MAIN_RAIN_ICE_OLD
 
   D = D0
 
-!$acc parallel loop gang vector private (YLSTACK, IBL, JLON, D, ISIZEMICRO) collapse (2)
+!$acc parallel loop gang vector present (PHYEX) private (YLSTACK, IBL, JLON, D, ISIZEMICRO) collapse (2)
 
   DO IBL = JBLK1, JBLK2
 
@@ -258,8 +270,14 @@ PROGRAM MAIN_RAIN_ICE_OLD
 
 #ifdef USE_STACK
     !Using cray pointers, AROME mechanism
+#if defined(USE_COLCALL) && defined(_OPENACC)
+    !Due to the collapse (2) directive, each point must have its own stack
+    YLSTACK%L = LOC (PSTACK (1, IBL)) + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8 * (JLON - 1)
+    YLSTACK%U = YLSTACK%L + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8
+#else
     YLSTACK%L = LOC (PSTACK (1, IBL))
     YLSTACK%U = YLSTACK%L + ISTSZ * KIND (PSTACK)
+#endif
 #else
     !Using fortran indexing, Meso-NH mechanism
     YLSTACK%L = 1
@@ -316,6 +334,10 @@ PROGRAM MAIN_RAIN_ICE_OLD
 
 !$acc end data
 
+#ifdef _OPENACC
+  CALL WIPE_PHYEX_T(PHYEX)
+#endif
+
   TED = OMP_GET_WTIME ()
 
   ZTC = ZTC + (TEC - TSC)

diff --git a/src/testprogs/shallow/main_shallow.F90 b/src/testprogs/shallow/main_shallow.F90
@@ -10,6 +10,9 @@ PROGRAM MAIN_SHALLOW
 USE MODE_MNH_ZWORK,  ONLY: ZMNH_STACK, IMNH_BLOCK, YMNH_STACK, INUMPIN
 USE OMP_LIB
 USE YOMHOOK, ONLY : LHOOK, DR_HOOK, JPHOOK
+#ifdef _OPENACC
+USE MODD_UTIL_PHYEX_T, ONLY: COPY_PHYEX_T, WIPE_PHYEX_T
+#endif
 
 IMPLICIT NONE
 
@@ -205,7 +208,11 @@ PROGRAM MAIN_SHALLOW
 D0%NIEC = D0%NIE
 D0%NJEC = D0%NJT
 
+#if defined(USE_COLCALL) && defined(_OPENACC)
+ISTSZ = NPROMA * 3000 * KLEV
+#else
 ISTSZ = NPROMA * 100 * KLEV
+#endif
 ALLOCATE (PSTACK (ISTSZ, NGPBLKS))
 ZMNH_STACK => PSTACK
 
@@ -220,8 +227,12 @@ PROGRAM MAIN_SHALLOW
 
   TSD = OMP_GET_WTIME ()
 
+#ifdef _OPENACC
+  CALL COPY_PHYEX_T(PHYEX)
+#endif
+
 !$acc data &
-!$acc      & copyin  (D0, PHYEX, &
+!$acc      & copyin  (D0, &
 !$acc      &          PDZZF, PZZ, PRHODJ, PRHODREF, PPABSM, PEXNM, PSFTH, PSFRV, PTHM, PRM, PUM, PVM, &
 !$acc      &          PTKEM, PSVM) & 
 !$acc      & copy    (PTHL_UP, PRT_UP, PRV_UP, PRC_UP, PRI_UP, PU_UP, PV_UP, PTHV_UP, PW_UP, PFRAC_UP, PEMF) &
@@ -245,7 +256,7 @@ PROGRAM MAIN_SHALLOW
 
   D = D0
 
-!$acc parallel loop gang vector private (YLSTACK, IBL, JLON, D) collapse (2)
+!$acc parallel loop gang vector present (PHYEX) private (YLSTACK, IBL, JLON, D) collapse (2)
 
   DO IBL = JBLK1, JBLK2
 
@@ -262,8 +273,14 @@ PROGRAM MAIN_SHALLOW
 
 #ifdef USE_STACK
     !Using cray pointers, AROME mechanism
+#if defined(USE_COLCALL) && defined(_OPENACC)
+    !Due to the collapse (2) directive, each point must have its own stack
+    YLSTACK%L = LOC (PSTACK (1, IBL)) + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8 * (JLON - 1)
+    YLSTACK%U = YLSTACK%L + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8
+#else
     YLSTACK%L = LOC (PSTACK (1, IBL))
     YLSTACK%U = YLSTACK%L + ISTSZ * KIND (PSTACK)
+#endif
 #else
     !Using fortran indexing, Meso-NH mechanism
     YLSTACK%L = 1
@@ -314,6 +331,10 @@ PROGRAM MAIN_SHALLOW
 
 !$acc end data
 
+#ifdef _OPENACC
+  CALL WIPE_PHYEX_T(PHYEX)
+#endif
+
   TED = OMP_GET_WTIME ()
 
   ZTC = ZTC + (TEC - TSC)

diff --git a/src/testprogs/turb_mnh/main_turb.F90 b/src/testprogs/turb_mnh/main_turb.F90
@@ -11,6 +11,9 @@ PROGRAM MAIN_TURB
 USE OMP_LIB
 USE YOMHOOK, ONLY : LHOOK, DR_HOOK, JPHOOK
 USE MODD_LES,        ONLY: TLES_t
+#ifdef _OPENACC
+USE MODD_UTIL_PHYEX_T, ONLY: COPY_PHYEX_T, WIPE_PHYEX_T
+#endif
 
 IMPLICIT NONE
 
@@ -222,7 +225,11 @@ PROGRAM MAIN_TURB
 D0%NIEC = D0%NIE
 D0%NJEC = D0%NJT
 
+#if defined(USE_COLCALL) && defined(_OPENACC)
+ISTSZ = NPROMA * 6000 * KLEV
+#else
 ISTSZ = NPROMA * 200 * KLEV
+#endif
 ALLOCATE (PSTACK (ISTSZ, NGPBLKS))
 ZMNH_STACK => PSTACK
 
@@ -237,8 +244,12 @@ PROGRAM MAIN_TURB
 
   TSD = OMP_GET_WTIME ()
 
+#ifdef _OPENACC
+  CALL COPY_PHYEX_T(PHYEX)
+#endif
+
 !$acc data &
-!$acc      & copyin  (D0, TLES, PHYEX, &
+!$acc      & copyin  (D0, TLES, &
 !$acc      &          ZDXX, ZDYY, ZDZZ, ZDZX, ZDZY, ZZZ, ZDIRCOSXW, ZDIRCOSYW, ZDIRCOSZW, ZCOSSLOPE, ZSINSLOPE,  &
 !$acc      &          PRHODJ, PTHVREF, PHGRAD, PZS, PSFTH, PSFRV, PSFSV, PSFU, PSFV, &
 !$acc      &          PPABSM, PUM, PVM, PWM, PTKEM, ZSVM, PSRCM, PLENGTHM, PLENGTHH, MFMOIST, &
@@ -264,7 +275,7 @@ PROGRAM MAIN_TURB
 
   D = D0
 
-!$acc parallel loop gang vector private (YLSTACK, IBL, JLON, D) collapse (2)
+!$acc parallel loop gang vector present (PHYEX) private (YLSTACK, IBL, JLON, D) collapse (2)
 
   DO IBL = JBLK1, JBLK2
 
@@ -281,8 +292,14 @@ PROGRAM MAIN_TURB
 
 #ifdef USE_STACK
     !Using cray pointers, AROME mechanism
+#if defined(USE_COLCALL) && defined(_OPENACC)
+    !Due to the collapse (2) directive, each point must have its own stack
+    YLSTACK%L = LOC (PSTACK (1, IBL)) + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8 * (JLON - 1)
+    YLSTACK%U = YLSTACK%L + CEILING(ISTSZ * KIND (PSTACK) / NPROMA / 8.) * 8
+#else
     YLSTACK%L = LOC (PSTACK (1, IBL))
     YLSTACK%U = YLSTACK%L + ISTSZ * KIND (PSTACK)
+#endif
 #else
     !Using fortran indexing, Meso-NH mechanism
     YLSTACK%L = 1

diff --git a/tools/INSTALL.sh b/tools/INSTALL.sh
@@ -4,7 +4,7 @@
 set -e
 set -o pipefail #abort if left command on a pipe fails
 
-pyft_version=fd212b8af8b74a1b31345966ae63eebe53473f00
+pyft_version=417286f52740ce558fd1dc9e6cfb0d35467903c5
 
 #This script installs PHYEX
 #Call the script with the -h option to get more information.