Merge pull request #60 from ecmwf-ifs/nab_atlas_MultiField

use Atlas structures with an IFS-like variable batching

README.md

@@ -96,6 +96,9 @@ Balthasar Reuter ([email protected])
   minor modifications (i.e. derived types/global paramters handling).
   Turned off by default, activate at the build stage with 
   `--cloudsc-fortran-pyiface=ON`.
+- **dwarf-cloudsc-fortran-atlas**: A version of **dwarf-cloudsc-fortran** which uses the [Atlas library](https://github.com/ecmwf/atlas) 
+  and its Field and FieldSet data stuctures. There are two storage settings for variables. If the environment variable
+  CLOUDSC_ATLAS_MULTIFIELD is "0", "OFF", or "FALSE", the variables are managed as atlas::FieldSet, which is an array of atlas::Fields. For other values of CLOUDSC_ATLAS_MULTIFIELD, a batching of variables is used as (BLK_IDX, LEV, VAR_ID, BLK_ID).
 
 
 ## Download and Installation

bundle.yml

@@ -70,7 +70,7 @@ projects :
 
     - atlas :
         git     : https://github.com/ecmwf/atlas
-        version : 0.34.0
+        version : feature/MultiField
         optional: true
         require : ecbuild eckit fckit
         cmake   : >

src/cloudsc_fortran_atlas/cloudsc_driver_mod.F90

@@ -45,6 +45,9 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, KFLDX, PTSPHY)
     INTEGER(KIND=JPIB) :: ENERGY, POWER, POWER_TOTAL, POWER_MAX, POWER_COUNT
     LOGICAL            :: LEC_PMON = .FALSE.
     CHARACTER(LEN=1)   :: CLEC_PMON
+    TYPE(ATLAS_TRACE)  :: TRACE
+
+    TRACE = ATLAS_TRACE("cloudsc_driver_mod.F90", __LINE__, "CLOUDSC_DRIVER","COMPUTE")
 
     CALL GET_ENVIRONMENT_VARIABLE('EC_PMON', CLEC_PMON)
     IF (CLEC_PMON == '1') LEC_PMON = .TRUE.
@@ -53,9 +56,8 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, KFLDX, PTSPHY)
     POWER_TOTAL = 0_JPIB
     POWER_COUNT = 0_JPIB
 
-    FIELD = FSET%FIELD(1)
+    FIELD = FSET%FIELD("PCLV")
     FSPACE = FIELD%FUNCTIONSPACE()
-    !NPROMA = FSPACE%NPROMA()
     NPROMA = FIELD%SHAPE(1)
     NLEV = FSPACE%LEVELS()
     NGPTOT = FSPACE%SIZE()
@@ -146,7 +148,11 @@ SUBROUTINE CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, KFLDX, PTSPHY)
          & (REAL(POWER_TOTAL, KIND=JPRD) / REAL(POWER_COUNT, KIND=JPRD)), &
          & "count:", POWER_COUNT
       END IF
-
+
+      CALL FIELD%FINAL()
+      CALL FSPACE%FINAL()
+      CALL TRACE%FINAL()
+
   END SUBROUTINE CLOUDSC_DRIVER
 
 END MODULE CLOUDSC_DRIVER_MOD

src/cloudsc_fortran_atlas/cloudsc_global_atlas_state_mod.F90

@@ -24,6 +24,7 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD
   USE ATLAS_MODULE
   USE, INTRINSIC :: ISO_C_BINDING
   USE ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS_MODULE
+  USE ATLAS_MULTIFIELD_MODULE
 
   IMPLICIT NONE
 
@@ -47,9 +48,6 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD
       "PRAINFRAC_TOPRFZ" /) 
 
   TYPE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW
-    TYPE(VAR2D_PTR), DIMENSION(24) :: IN_VARS_2D_REAL64
-    TYPE(VAR2D_PTR), DIMENSION(15) :: OUT_VARS_2D_REAL64
-
     ! Input field variables and tendencies
     REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PLCRIT_AER(:,:)
     REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PICRIT_AER(:,:) 
@@ -77,8 +75,8 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD
     REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PSUPSAT(:,:)
 
     REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PLSM(:)       ! Land fraction (0-1) 
-    LOGICAL,        POINTER, CONTIGUOUS :: LDCUM(:)      ! Convection active
-    INTEGER,        POINTER, CONTIGUOUS :: KTYPE(:)      ! Convection type 0,1,2
+    LOGICAL,         POINTER, CONTIGUOUS :: LDCUM(:)      ! Convection active
+    INTEGER,         POINTER, CONTIGUOUS :: KTYPE(:)      ! Convection type 0,1,2
     REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PAPH(:,:)     ! Pressure on half levels
     REAL(KIND=JPRB), POINTER, CONTIGUOUS :: PCLV(:,:,:) 
 
@@ -129,7 +127,7 @@ MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD
 
   SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK)
     CLASS(CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK_VIEW), INTENT(INOUT) :: SELF
-    CLASS(ATLAS_FIELDSET), INTENT(INOUT) :: FSET
+    TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET
     INTEGER, INTENT(IN) :: IBLK
 
     REAL(KIND=JPRB), POINTER :: TMP3D(:,:,:)
@@ -190,73 +188,135 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK(SELF, FSET, IBLK)
     CALL FSET%DATA(40, SELF%PFCQSNG, IBLK)
     CALL FSET%DATA(41, SELF%PFSQLTUR, IBLK)
     CALL FSET%DATA(42, SELF%PFSQITUR, IBLK)
-    CALL FSET%DATA(43,SELF%PFPLSL, IBLK)
+    CALL FSET%DATA(43, SELF%PFPLSL, IBLK)
     CALL FSET%DATA(44, SELF%PFPLSN, IBLK)
     CALL FSET%DATA(45, SELF%PFHPSL, IBLK)
     CALL FSET%DATA(46, SELF%PFHPSN, IBLK)
     CALL FSET%DATA(47, SELF%PCOVPTOT, IBLK)
     CALL FSET%DATA(48, SELF%PRAINFRAC_TOPRFZ, IBLK)
   END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_BLOCK
 
-  SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG)
+  SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, FSPACE, NPROMA, NGPTOTG)
     ! Load reference input data via serialbox
     CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF
     TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET
-    INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA
-    INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG
+    TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS), INTENT(INOUT) :: FSPACE
+    INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA, NGPTOTG
 
     TYPE(ATLAS_STRUCTUREDGRID) :: GRID
-    TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE
+    TYPE(ATLAS_MULTIFIELD) :: MULTIFIELD_IN, MULTIFIELD_OUT
     INTEGER(KIND=JPIM) :: KLON, IVAR, B
-    TYPE(VAR3D_PTR), DIMENSION(24) :: IN_VARS_3D_REAL64
-    TYPE(VAR3D_PTR), DIMENSION(15) :: OUT_VARS_3D_REAL64
     REAL(KIND=JPRB), POINTER :: TMP3D(:,:,:)
     REAL(KIND=JPRB), POINTER :: TMP2D(:,:)
     TYPE(ATLAS_FIELD) :: FIELD
-    TYPE(ATLAS_PARTITIONER) :: PARTITIONER
-    INTEGER(KIND=JPIM)      :: NGPTOT
+    TYPE(ATLAS_CONFIG) :: CONFIG
+    TYPE(ATLAS_CONFIG), DIMENSION(24) :: IN_MFIELD_CONFIG ! the last five variables are special and added through FieldSet
+    TYPE(ATLAS_CONFIG), DIMENSION(14) :: OUT_MFIELD_CONFIG ! the last two variables are special and added through FieldSet
+    INTEGER :: INVAR_SIZE, OUTVAR_SIZE
+    LOGICAL :: LMULTIFIELD
+    CHARACTER(len=8) :: CENV, FPREC
+    INTEGER :: CENV_LEN
+    TYPE(ATLAS_TRACE) :: TRACE, TRACE_IO
+
+    TRACE = ATLAS_TRACE("cloudsc_global_atlas_state_mod.F90", __LINE__, &
+        & "CLOUDSC_GLOBAL_ATLAS_STATE_LOAD")
+
+    LMULTIFIELD = .TRUE.
+    CALL GET_ENVIRONMENT_VARIABLE("CLOUDSC_ATLAS_MULTIFIELD",CENV,CENV_LEN)
+    IF (CENV_LEN > 0 ) THEN
+      IF( TRIM(CENV) == "0" .OR. TRIM(CENV) == "OFF" .OR. TRIM(CENV) == "FALSE" ) THEN
+        LMULTIFIELD = .FALSE.
+      ENDIF
+    ENDIF
+
+    IF (IRANK == 0) THEN
+      PRINT *, " LMULTIFIELD: ", LMULTIFIELD
+    ENDIF
+
     CALL INPUT_INITIALIZE(NAME='input')
 
     CALL LOAD_SCALAR('KLON', KLON)
     CALL LOAD_SCALAR('KLEV', SELF%KLEV)
     CALL LOAD_SCALAR('KFLDX', SELF%KFLDX)
 
-    GRID = ATLAS_REGULARLONLATGRID(NGPTOTG,1)
+    GRID = ATLAS_REGULARLONLATGRID(NGPTOTG, 1)
     FSPACE = ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS(GRID, LEVELS=SELF%KLEV, NPROMA=NPROMA, HALO=0)
-    NGPTOT = FSPACE%SIZE()
     SELF%NBLOCKS = FSPACE%NBLKS()
-    FSET = ATLAS_FIELDSET()
 
-    DO IVAR = 1, SIZE(IN_VAR_NAMES) - 5 ! the last variables are special
-        CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(IN_VAR_NAMES(IVAR)), KIND=ATLAS_REAL(JPRB)))
-    ENDDO
+    ! create multifield
+    CONFIG = ATLAS_CONFIG()
+    CALL CONFIG%SET("type", "MultiFieldCreatorIFS")
+    CALL CONFIG%SET("nproma", NPROMA)
+    CALL CONFIG%SET("ngptot", FSPACE%SIZE())
+    WRITE (FPREC,"(A4,I2)") "real", 8*jprb
+    CALL CONFIG%SET("datatype", TRIM(FPREC))
+
+    INVAR_SIZE = SIZE(IN_VAR_NAMES) - 5
+    IF (LMULTIFIELD) THEN
+      ! input multifield on model levels, i.e. LEVELS = FSPACE%LEVELS() = SELF%KLEV
+      DO IVAR = 1, INVAR_SIZE
+          IN_MFIELD_CONFIG(IVAR) = ATLAS_CONFIG()
+          CALL IN_MFIELD_CONFIG(IVAR)%SET("name", TRIM(IN_VAR_NAMES(IVAR)))
+      END DO
+      CALL CONFIG%SET("nlev", FSPACE%LEVELS())
+      CALL CONFIG%SET("fields", IN_MFIELD_CONFIG)
+      DO IVAR = 1, INVAR_SIZE
+          CALL IN_MFIELD_CONFIG(IVAR)%FINAL()
+      END DO
+      MULTIFIELD_IN = atlas_MultiField(CONFIG)
+      CALL FSET%ADD(MULTIFIELD_IN%FIELDSET()) 
+      CALL MULTIFIELD_IN%FINAL()
+    ELSE
+      DO IVAR = 1, INVAR_SIZE
+          CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(IN_VAR_NAMES(IVAR)),   KIND=ATLAS_REAL(JPRB)))
+      END DO
+    ENDIF
+
+    ! special six input variables
     CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PLSM",   KIND=ATLAS_REAL(JPRB),    LEVELS=0))
     CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="LDCUM",  KIND=ATLAS_LOGICAL(),     LEVELS=0))
     CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="KTYPE",  KIND=ATLAS_INTEGER(JPIM), LEVELS=0))
     CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PAPH",   KIND=ATLAS_REAL(JPRB),    LEVELS=SELF%KLEV+1))
-    !CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PEXTRA", KIND=ATLAS_REAL(JPRB),    VARIABLES=SELF%KFLDX))
+    !CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PEXTRA", KIND=ATLAS_REAL(JPRB),    VARIABLES=MAX(1,SELF%KFLDX)))
     CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PCLV",   KIND=ATLAS_REAL(JPRB),    VARIABLES=MAX(1,NCLV)))
+    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME='TENDENCY_CML', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV))
+    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME='TENDENCY_TMP', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV))
+    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME='TENDENCY_LOC', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV))
+
+
+    OUTVAR_SIZE = SIZE(OUT_VAR_NAMES) - 2
+    IF (LMULTIFIELD) THEN
+        ! output multifield on model interfaces, i.e. LEVELS = FSPACE%LEVELS() + 1 = SELF%KLEV + 1
+        DO IVAR = 1, OUTVAR_SIZE
+            OUT_MFIELD_CONFIG(IVAR) = ATLAS_CONFIG()
+            CALL OUT_MFIELD_CONFIG(IVAR)%SET("name", TRIM(OUT_VAR_NAMES(IVAR)))
+            CALL OUT_MFIELD_CONFIG(IVAR)%SET("levels", SELF%KLEV+1)
+        END DO
+        CALL CONFIG%SET("nlev", 1 + FSPACE%LEVELS())
+        CALL CONFIG%SET("fields", OUT_MFIELD_CONFIG)
+        DO IVAR = 1, OUTVAR_SIZE
+            CALL OUT_MFIELD_CONFIG(IVAR)%FINAL()
+        END DO
+        MULTIFIELD_OUT = atlas_MultiField(CONFIG)
+        CALL FSET%ADD(MULTIFIELD_OUT%FIELDSET()) 
+        CALL MULTIFIELD_OUT%FINAL()
+    ELSE
+      DO IVAR = 1, OUTVAR_SIZE
+          CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(OUT_VAR_NAMES(IVAR)),   KIND=ATLAS_REAL(JPRB), LEVELS=SELF%KLEV+1))
+      END DO
+    ENDIF
+
+    ! special two output variables
+    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PCOVPTOT", KIND=ATLAS_REAL(JPRB)))
+    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PRAINFRAC_TOPRFZ", KIND=ATLAS_REAL(JPRB), LEVELS=0))
 
     DO IVAR = 1, SIZE(IN_VAR_NAMES)
-        CALL LOADVAR_ATLAS(FSET, TRIM(IN_VAR_NAMES(IVAR)), KLON, NGPTOTG)
+        CALL LOADVAR_ATLAS(FSET, FSPACE, TRIM(IN_VAR_NAMES(IVAR)), KLON, NGPTOTG)
     ENDDO
-
-    FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_CML', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV)
-    CALL FSET%ADD(FIELD)
-    FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_TMP', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV)
-    CALL FSET%ADD(FIELD)
-    FIELD = FSPACE%CREATE_FIELD(NAME='TENDENCY_LOC', KIND=ATLAS_REAL(JPRB), VARIABLES=3+NCLV)
-    CALL FSET%ADD(FIELD)
-
     CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_CML', KLON, NGPTOTG)
     CALL LOADSTATE_ATLAS(FSET, 'TENDENCY_TMP', KLON, NGPTOTG)
-    ! Output fields are simply allocated and zero'd
-    DO IVAR = 1, SIZE(OUT_VAR_NAMES) - 2
-        CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME=TRIM(OUT_VAR_NAMES(IVAR)), KIND=ATLAS_REAL(JPRB), LEVELS=SELF%KLEV+1))
-    ENDDO
-    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PCOVPTOT", KIND=ATLAS_REAL(JPRB)))
-    CALL FSET%ADD(FSPACE%CREATE_FIELD(NAME="PRAINFRAC_TOPRFZ", KIND=ATLAS_REAL(JPRB), LEVELS=0))
 
+    ! Output fields are simply zero'd
     DO IVAR = 1, SIZE(OUT_VAR_NAMES) - 1
         FIELD = FSET%FIELD(TRIM(OUT_VAR_NAMES(IVAR)))
         CALL FIELD%DATA(TMP3D)
@@ -285,16 +345,24 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD(SELF, FSET, NPROMA, NGPTOTG)
     CALL YREPHLI_LOAD_PARAMETERS()
 
     CALL INPUT_FINALIZE()
+
+    CALL FIELD%FINAL()
+    CALL CONFIG%FINAL()
+    CALL GRID%FINAL()
+    CALL TRACE%FINAL()
   END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_LOAD
 
-  SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOTG)
+  SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, FSPACE, NGPTOTG)
     ! Validate the correctness of output against reference data
     CLASS(CLOUDSC_GLOBAL_ATLAS_STATE) :: SELF
     TYPE(ATLAS_FIELDSET), INTENT(INOUT) :: FSET
-    INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: NGPTOTG
+    TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS), INTENT(IN) :: FSPACE
+    INTEGER(KIND=JPIM), INTENT(IN) :: NGPTOTG
 
+    TYPE(ATLAS_TRACE) :: TRACE
     INTEGER(KIND=JPIM) :: KLON, IVAR
 
+    TRACE = ATLAS_TRACE("cloudsc_global_atlas_state_mod.F90", __LINE__, "VALIDATE", "VALIDATE")
     CALL INPUT_INITIALIZE(NAME='reference')
     CALL LOAD_SCALAR('KLON', KLON)
     print *, "KLON = ", KLON
@@ -307,13 +375,13 @@ SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE(SELF, FSET, NGPTOTG)
 
 
     ! Actual variable validation
-    CALL VALIDATEVAR_ATLAS(FSET, 'PLUDE', KLON, NGPTOTG)
+    CALL VALIDATEVAR_ATLAS(FSET, FSPACE, 'PLUDE', KLON, NGPTOTG)
     DO IVAR = 1, SIZE(OUT_VAR_NAMES)
-        CALL VALIDATEVAR_ATLAS(FSET, OUT_VAR_NAMES(IVAR), KLON, NGPTOTG)
+        CALL VALIDATEVAR_ATLAS(FSET, FSPACE, TRIM(OUT_VAR_NAMES(IVAR)), KLON, NGPTOTG)
     ENDDO
-    CALL VALIDATESTATE_ATLAS(FSET, 'TENDENCY_LOC', KLON, NGPTOTG)
+    CALL VALIDATESTATE_ATLAS(FSET, FSPACE, 'TENDENCY_LOC', KLON, NGPTOTG)
     CALL INPUT_FINALIZE()
-
+    CALL TRACE%FINAL()
   END SUBROUTINE CLOUDSC_GLOBAL_ATLAS_STATE_VALIDATE
 
 END MODULE CLOUDSC_GLOBAL_ATLAS_STATE_MOD

src/cloudsc_fortran_atlas/dwarf_cloudsc_atlas.F90

@@ -17,6 +17,8 @@ PROGRAM DWARF_CLOUDSC
 
 USE ATLAS_MODULE
 USE, INTRINSIC :: ISO_C_BINDING
+USE ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS_MODULE
+USE ATLAS_MULTIFIELD_MODULE
 
 IMPLICIT NONE
 
@@ -27,14 +29,14 @@ PROGRAM DWARF_CLOUDSC
 INTEGER(KIND=JPIM) :: NGPTOTG  = 16384 ! Number of grid points (as read from command line)
 INTEGER(KIND=JPIM) :: NPROMA   = 32    ! NPROMA blocking factor (currently active)
 
-type(atlas_fieldset) :: fset
-type(atlas_field) :: field
-
+TYPE(ATLAS_FIELDSET) :: FSET
+TYPE(ATLAS_FUNCTIONSPACE_BLOCKSTRUCTUREDCOLUMNS) :: FSPACE
 TYPE(CLOUDSC_GLOBAL_ATLAS_STATE) :: GLOBAL_ATLAS_STATE
-
+TYPE(ATLAS_TRACE) :: TRACE
 INTEGER(KIND=JPIB) :: ENERGY, POWER
 CHARACTER(LEN=1)   :: CLEC_PMON
 
+
 CALL GET_ENVIRONMENT_VARIABLE('EC_PMON', CLEC_PMON)
 IF (CLEC_PMON == '1') THEN
   CALL EC_PMON(ENERGY, POWER)
@@ -52,6 +54,7 @@ PROGRAM DWARF_CLOUDSC
 ! Initialize MPI environment
 CALL CLOUDSC_MPI_INIT(NUMOMP)
 CALL ATLAS_LIBRARY%INITIALISE()
+TRACE = ATLAS_TRACE("dwarf_cloudsc_atlas.F90",__LINE__,"program")
 
 ! Get total number of grid points (NGPTOTG) with which to run the benchmark
 IF (IARGS >= 2) THEN
@@ -68,13 +71,18 @@ PROGRAM DWARF_CLOUDSC
 FSET = ATLAS_FIELDSET()
 
 ! TODO: Create a global memory state from serialized input data
-CALL GLOBAL_ATLAS_STATE%LOAD(FSET, NPROMA, NGPTOTG)
+CALL GLOBAL_ATLAS_STATE%LOAD(FSET, FSPACE, NPROMA, NGPTOTG)
 
 ! Call the driver to perform the parallel loop over our kernel
 CALL CLOUDSC_DRIVER(FSET, NUMOMP, NGPTOTG, GLOBAL_ATLAS_STATE%KFLDX, GLOBAL_ATLAS_STATE%PTSPHY)
 
 ! Validate the output against serialized reference data
-CALL GLOBAL_ATLAS_STATE%VALIDATE(FSET, NGPTOTG)
+CALL GLOBAL_ATLAS_STATE%VALIDATE(FSET, FSPACE, NGPTOTG)
+
+CALL FSET%FINAL()
+CALL FSPACE%FINAL()
+
+CALL TRACE%FINAL()
 
 ! Tear down MPI environment
 CALL ATLAS_LIBRARY%FINALISE()