From 7721be5675b0f06ddf411d78fc2619075c99392e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 14:29:46 +0100 Subject: [PATCH] Add icon async case --- cases/icon-async-test/config.yaml | 52 +++ cases/icon-async-test/icon_runjob.cfg | 345 ++++++++++++++++++ .../icontools_remap_00_lbc_runjob.cfg | 154 ++++++++ .../icontools_remap_ic_runjob.cfg | 245 +++++++++++++ .../icontools_remap_lbc_rest_runjob.cfg | 128 +++++++ 5 files changed, 924 insertions(+) create mode 100644 cases/icon-async-test/config.yaml create mode 100755 cases/icon-async-test/icon_runjob.cfg create mode 100755 cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg create mode 100755 cases/icon-async-test/icontools_remap_ic_runjob.cfg create mode 100755 cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg diff --git a/cases/icon-async-test/config.yaml b/cases/icon-async-test/config.yaml new file mode 100644 index 00000000..1000bebf --- /dev/null +++ b/cases/icon-async-test/config.yaml @@ -0,0 +1,52 @@ +# Configuration file for the 'icon-async-test' case with ICON + +workflow: icon-async +constraint: gpu +run_on: cpu +compute_queue: normal +ntasks_per_node: 12 +restart_step: PT6H +startdate: 2018-01-01T00:00:00Z +enddate: 2018-01-01T12:00:00Z + +eccodes_dir: ./input/eccodes_definitions +iconremap_bin: iconremap +iconsub_bin: iconsub +latbc_filename: ifs_201801_lbc.nc +inidata_filename: ifs_init_2018010100.nc +output_filename: NWP_LAM +filename_format: _DOM_ +lateral_boundary_grid_order: lateral_boundary + +meteo: + dir: ./input/meteo + prefix: ifs_ + nameformat: '%Y%m%d%H' + suffix: .grb + inc: 3 + +icontools_runjobs: + - icontools_remap_ic_runjob.cfg + - icontools_remap_00_lbc_runjob.cfg + - icontools_remap_lbc_rest_runjob.cfg + +input_files: + radiation_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.parent.nc + dynamics_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.nc + map_file_latbc: ./input/icon/grid/map_file.latbc + lateral_boundary_grid: ./input/icon/grid/lateral_boundary.grid.nc + extpar_filename: ./input/icon/grid/external_parameter_icon_VERIFY_DOM_DOM01_tiles.nc + cldopt_filename: ./input/icon/rad/rrtm_cldopt.nc + lrtm_filename: ./input/icon/rad/rrtmg_lw.nc + map_file_ana: ./input/icon/mapping/map_file.ana + +icon: + binary_file: ./ext/icon/bin/icon + runjob_filename: icon_runjob.cfg + compute_queue: normal + walltime: '00:10:00' + np_tot: 8 + np_io: 1 + np_restart: 1 + np_prefetch: 1 + diff --git a/cases/icon-async-test/icon_runjob.cfg b/cases/icon-async-test/icon_runjob.cfg new file mode 100755 index 00000000..09ff2c3a --- /dev/null +++ b/cases/icon-async-test/icon_runjob.cfg @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --time={cfg.icon_walltime} +#SBATCH --nodes={cfg.icon_np_tot} +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --hint=nomultithread +#SBATCH --output={logfile} +#SBATCH --open-mode=append +#SBATCH --chdir={cfg.icon_work} + +# OpenMP environment variables +# ---------------------------- +export OMP_NUM_THREADS=1 +export ICON_THREADS=1 +export OMP_SCHEDULE=static,12 +export OMP_DYNAMIC="false" +export OMP_STACKSIZE=200M +# +# MPI variables +# ------------- +no_of_nodes={cfg.icon_np_tot} +mpi_procs_pernode={cfg.ntasks_per_node} +((mpi_total_procs=no_of_nodes * mpi_procs_pernode)) +# +# blocking length +# --------------- +nproma=16 + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +# ---------------------------------------------------------------------------- +# create ICON master namelist +# ---------------------------------------------------------------------------- + +cat > icon_master.namelist << EOF +! master_nml: ---------------------------------------------------------------- +&master_nml + lrestart = {cfg.lrestart} ! .TRUE.=current experiment is resumed + read_restart_namelists = .true. +/ + +! master_time_control_nml: --------------------------------------------------- +&master_time_control_nml + calendar = 'proleptic gregorian' + restartTimeIntval = '{cfg.restart_step}' + checkpointTimeIntval = '{cfg.restart_step}' + experimentStartDate = '{cfg.ini_datetime_string}' + experimentStopDate = '{cfg.end_datetime_string}' +/ + +! master_model_nml: repeated for each model ---------------------------------- +&master_model_nml + model_type = 1 ! identifies which component to run (atmosphere,ocean,...) + model_name = "ATMO" ! character string for naming this component. + model_namelist_filename = "NAMELIST_{cfg.casename}" ! file name containing the model namelists + model_min_rank = 1 ! start MPI rank for this model + model_max_rank = 65536 ! end MPI rank for this model + model_inc_rank = 1 ! stride of MPI ranks +/ +EOF + +# ---------------------------------------------------------------------- +# model namelists +# ---------------------------------------------------------------------- + +cat > NAMELIST_{cfg.casename} << EOF +! parallel_nml: MPI parallelization ------------------------------------------- +¶llel_nml + nproma = 128 ! loop chunk length + p_test_run = .FALSE. ! .TRUE. means verification run for MPI parallelization + num_io_procs = {cfg.icon_np_io} ! number of I/O processors + num_restart_procs = {cfg.icon_np_restart} ! number of restart processors + num_prefetch_proc = {cfg.icon_np_prefetch} ! number of processors for LBC prefetching + iorder_sendrecv = 3 ! sequence of MPI send/receive calls +/ + + +! run_nml: general switches --------------------------------------------------- +&run_nml + ltestcase = .FALSE. ! real case run + num_lev = 60 ! number of full levels (atm.) for each domain + lvert_nest = .FALSE. ! no vertical nesting + dtime = 60. ! timestep in seconds + ldynamics = .TRUE. ! compute adiabatic dynamic tendencies + ltransport = .TRUE. ! compute large-scale tracer transport + ntracer = 0 ! number of advected tracers + iforcing = 3 ! forcing of dynamics and transport by parameterized processes + msg_level = 13 ! detailed report during integration + ltimer = .TRUE. ! timer for monitoring the runtime of specific routines + timers_level = 10 ! performance timer granularity + check_uuid_gracefully = .TRUE. ! give only warnings for non-matching uuids + output = "nml" ! main switch for enabling/disabling components of the model output + lart = .FALSE. ! main switch for ART + debug_check_level = 10 + restart_filename = "{cfg.icon_restart_out}/{cfg.output_filename}_.nc" + activate_sync_timers = .TRUE. +/ + +! diffusion_nml: horizontal (numerical) diffusion ---------------------------- +&diffusion_nml + lhdiff_vn = .TRUE. ! diffusion on the horizontal wind field + lhdiff_temp = .TRUE. ! diffusion on the temperature field + lhdiff_w = .TRUE. ! diffusion on the vertical wind field + hdiff_order = 5 ! order of nabla operator for diffusion + itype_vn_diffu = 1 ! reconstruction method used for Smagorinsky diffusion + itype_t_diffu = 2 ! discretization of temperature diffusion + hdiff_efdt_ratio = 24.0 ! ratio of e-folding time to time step + hdiff_smag_fac = 0.025 ! scaling factor for Smagorinsky diffusion +/ + +! dynamics_nml: dynamical core ----------------------------------------------- +&dynamics_nml + iequations = 3 ! type of equations and prognostic variables + idiv_method = 1 ! method for divergence computation + divavg_cntrwgt = 0.50 ! weight of central cell for divergence averaging + lcoriolis = .TRUE. ! Coriolis force +/ + +! extpar_nml: external data -------------------------------------------------- +&extpar_nml + itopo = 1 ! topography (0:analytical) + extpar_filename = '{cfg.input_files_scratch_extpar_filename}' ! filename of external parameter input file + n_iter_smooth_topo = 1,1 ! iterations of topography smoother + heightdiff_threshold = 3000. ! height difference between neighb. grid points + hgtdiff_max_smooth_topo = 750.,750. ! see Namelist doc + heightdiff_threshold = 2250.,1500. +/ + +! initicon_nml: specify read-in of initial state ------------------------------ +&initicon_nml + init_mode = 2 ! 7: start from DWD fg with subsequent vertical remapping + lread_ana = .FALSE. ! no analysis data will be read + ifs2icon_filename = "{inidata_filename}" ! initial data filename + ana_varnames_map_file = "{cfg.input_files_scratch_map_file_ana}" ! dictionary mapping internal names onto GRIB2 shortNames + ltile_coldstart = .TRUE. ! coldstart for surface tiles + ltile_init = .FALSE. ! set it to .TRUE. if FG data originate from run without tiles +/ + +! grid_nml: horizontal grid -------------------------------------------------- +&grid_nml + dynamics_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" ! array of the grid filenames for the dycore + radiation_grid_filename = "{cfg.input_files_scratch_radiation_grid_filename}" ! array of the grid filenames for the radiation model + dynamics_parent_grid_id = 0 ! array of the indexes of the parent grid filenames + lredgrid_phys = .TRUE. ! .true.=radiation is calculated on a reduced grid + lfeedback = .TRUE. ! specifies if feedback to parent grid is performed + l_limited_area = .TRUE. ! .TRUE. performs limited area run + ifeedback_type = 2 ! feedback type (incremental/relaxation-based) + start_time = 0. ! Time when a nested domain starts to be active [s] +/ + +! gridref_nml: grid refinement settings -------------------------------------- +&gridref_nml + denom_diffu_v = 150. ! denominator for lateral boundary diffusion of velocity +/ + +! interpol_nml: settings for internal interpolation methods ------------------ +&interpol_nml + nudge_zone_width = 8 ! width of lateral boundary nudging zone + support_baryctr_intp = .FALSE. ! barycentric interpolation support for output + nudge_max_coeff = 0.07 + nudge_efold_width = 2.0 +/ + +! io_nml: general switches for model I/O ------------------------------------- +&io_nml + itype_pres_msl = 5 ! method for computation of mean sea level pressure + itype_rh = 1 ! method for computation of relative humidity + lmask_boundary = .TRUE. ! mask out interpolation zone in output + restart_file_type = 5 +/ + +! limarea_nml: settings for limited area mode --------------------------------- +&limarea_nml + itype_latbc = 1 ! 1: time-dependent lateral boundary conditions + dtime_latbc = 10800 ! time difference between 2 consecutive boundary data + nlev_latbc = 90 ! Number of vertical levels in boundary data + latbc_boundary_grid = "{cfg.input_files_scratch_lateral_boundary_grid}" ! Grid file defining the lateral boundary + latbc_path = "{cfg.icon_input_icbc}" ! Absolute path to boundary data + latbc_varnames_map_file = "{cfg.input_files_scratch_map_file_latbc}" + latbc_filename = "{cfg.latbc_filename}" ! boundary data input filename + init_latbc_from_fg = .FALSE. ! .TRUE.: take lbc for initial time from first guess +/ + +! lnd_nml: land scheme switches ----------------------------------------------- +&lnd_nml + ntiles = 1 ! number of tiles + nlev_snow = 3 ! number of snow layers + lmulti_snow = .FALSE. ! .TRUE. for use of multi-layer snow model + idiag_snowfrac = 20 ! type of snow-fraction diagnosis + lsnowtile = .TRUE. ! .TRUE.=consider snow-covered and snow-free separately + itype_root = 2 ! root density distribution + itype_heatcond = 3 ! type of soil heat conductivity + itype_lndtbl = 4 ! table for associating surface parameters + itype_evsl = 4 ! type of bare soil evaporation + itype_root = 2 ! root density distribution + cwimax_ml = 5.e-4 ! scaling parameter for max. interception storage + c_soil = 1.75 ! surface area density of the evaporative soil surface + c_soil_urb = 0.5 ! same for urban areas + lseaice = .TRUE. ! .TRUE. for use of sea-ice model + llake = .TRUE. ! .TRUE. for use of lake model +/ + +! nonhydrostatic_nml: nonhydrostatic model ----------------------------------- +&nonhydrostatic_nml + iadv_rhotheta = 2 ! advection method for rho and rhotheta + ivctype = 2 ! type of vertical coordinate + itime_scheme = 4 ! time integration scheme + ndyn_substeps = 5 ! number of dynamics steps per fast-physics step + exner_expol = 0.333 ! temporal extrapolation of Exner function + vwind_offctr = 0.2 ! off-centering in vertical wind solver + damp_height = 12500.0 ! height at which Rayleigh damping of vertical wind starts + rayleigh_coeff = 1.5 ! Rayleigh damping coefficient + divdamp_order = 24 ! order of divergence damping + divdamp_type = 3 ! type of divergence damping + divdamp_fac = 0.004 ! scaling factor for divergence damping + l_open_ubc = .FALSE. ! .TRUE.=use open upper boundary condition + igradp_method = 3 ! discretization of horizontal pressure gradient + l_zdiffu_t = .TRUE. ! specifies computation of Smagorinsky temperature diffusion + thslp_zdiffu = 0.02 ! slope threshold (temperature diffusion) + thhgtd_zdiffu = 125.0 ! threshold of height difference (temperature diffusion) + htop_moist_proc = 22500.0 ! max. height for moist physics + hbot_qvsubstep = 22500.0 ! height above which QV is advected with substepping scheme +/ + +! nwp_phy_nml: switches for the physics schemes ------------------------------ +&nwp_phy_nml + inwp_gscp = 2 ! cloud microphysics and precipitation + inwp_convection = 1 ! convection + lshallowconv_only = .FALSE. ! only shallow convection + inwp_radiation = 1 ! radiation + inwp_cldcover = 1 ! cloud cover scheme for radiation + inwp_turb = 1 ! vertical diffusion and transfer + inwp_satad = 1 ! saturation adjustment + inwp_sso = 1 ! subgrid scale orographic drag + inwp_gwd = 0 ! non-orographic gravity wave drag + inwp_surface = 1 ! surface scheme + latm_above_top = .TRUE. ! take into account atmosphere above model top for radiation computation + ldetrain_conv_prec = .TRUE. + efdt_min_raylfric = 7200. ! minimum e-folding time of Rayleigh friction + itype_z0 = 2 ! type of roughness length data + icapdcycl = 3 ! apply CAPE modification to improve diurnalcycle over tropical land + icpl_aero_conv = 1 ! coupling between autoconversion and Tegen aerosol climatology + icpl_aero_gscp = 1 ! coupling between autoconversion and Tegen aerosol climatology + lrtm_filename = '{cfg.input_files_scratch_lrtm_filename}' ! longwave absorption coefficients for RRTM_LW + cldopt_filename = '{cfg.input_files_scratch_cldopt_filename}' ! RRTM cloud optical properties + dt_rad = 720. ! time step for radiation in s + dt_conv = 120.,90.,90. ! time step for convection in s (domain specific) + dt_sso = 120.,360.,360. ! time step for SSO parameterization + dt_gwd = 360.,360.,360. ! time step for gravity wave drag parameterization +/ + +! nwp_tuning_nml: additional tuning parameters ---------------------------------- +&nwp_tuning_nml + itune_albedo = 1 ! reduced albedo (w.r.t. MODIS data) over Sahara + tune_gkwake = 1.8 + tune_gkdrag = 0.01 + tune_minsnowfrac = 0.3 +/ + +! output_nml: specifies an output stream -------------------------------------- +&output_nml + filetype = 4 ! output format: 2=GRIB2, 4=NETCDFv2 + dom = 1 ! write domain 1 only + output_bounds = 0., 10000000., 3600. ! start, end, increment + steps_per_file = 1 ! number of steps per file + mode = 1 ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis) + include_last = .TRUE. + output_filename = '{cfg.output_filename}' + filename_format = '{cfg.icon_output}/_DOM_' ! file name base + steps_per_file_inclfirst = .FALSE. + output_grid = .TRUE. + remap = 1 ! 1: remap to lat-lon grid + !north_pole = -170.,40. ! definition of north_pole for rotated lat-lon grid + reg_lon_def = -16.0,0.13,36.0 ! + reg_lat_def = 32.0,0.12,74.0 ! + ml_varlist = 'group:PBL_VARS', + 'group:ATMO_ML_VARS', + 'group:precip_vars', + 'group:land_vars', + 'group:nh_prog_vars', + 'z_mc', 'z_ifc', +/ + +! radiation_nml: radiation scheme --------------------------------------------- +&radiation_nml + irad_o3 = 7 ! ozone climatology + irad_aero = 6 ! aerosols + albedo_type = 2 ! type of surface albedo + vmr_co2 = 390.e-06 + vmr_ch4 = 1800.e-09 + vmr_n2o = 322.0e-09 + vmr_o2 = 0.20946 + vmr_cfc11 = 240.e-12 + vmr_cfc12 = 532.e-12 +/ + +! sleve_nml: vertical level specification ------------------------------------- +&sleve_nml + min_lay_thckn = 20.0 ! layer thickness of lowermost layer + top_height = 23000.0 ! height of model top + stretch_fac = 0.65 ! stretching factor to vary distribution of model levels + decay_scale_1 = 4000.0 ! decay scale of large-scale topography component + decay_scale_2 = 2500.0 ! decay scale of small-scale topography component + decay_exp = 1.2 ! exponent of decay function + flat_height = 16000.0 ! height above which the coordinate surfaces are flat +/ + +! transport_nml: tracer transport --------------------------------------------- +&transport_nml + npassive_tracer = 0 ! number of additional passive tracers + ivadv_tracer = 3, 3, 3, 3, 3, 3 ! tracer specific method to compute vertical advection + itype_hlimit = 3, 4, 4, 4, 4, 4 ! type of limiter for horizontal transport + ihadv_tracer = 52, 2, 2, 2, 2, 22 ! tracer specific method to compute horizontal advection + llsq_svd = .TRUE. ! use SV decomposition for least squares design matrix +/ + +! turbdiff_nml: turbulent diffusion ------------------------------------------- +&turbdiff_nml + tkhmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient + tkmmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient + pat_len = 750.0 ! effective length scale of thermal surface patterns + c_diff = 0.2 ! length scale factor for vertical diffusion of TKE + rat_sea = 7.5 ! controls laminar resistance for sea surface + rlam_heat = 1.5 + ltkesso = .TRUE. ! consider TKE-production by sub-grid SSO wakes + frcsmot = 0.2 ! these 2 switches together apply vertical smoothing of the TKE source terms + imode_frcsmot = 2 ! in the tropics (only), which reduces the moist bias in the tropical lower troposphere + itype_sher = 3 ! type of shear forcing used in turbulence + ltkeshs = .TRUE. ! include correction term for coarse grids in hor. shear production term + a_hshr = 2.0 ! length scale factor for separated horizontal shear mode + icldm_turb = 1 ! mode of cloud water representation in turbulence + ldiff_qi = .TRUE. +/ + +EOF + +# ---------------------------------------------------------------------- +# run the model! +# ---------------------------------------------------------------------- + srun ./icon.exe diff --git a/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg new file mode 100755 index 00000000..a93ac550 --- /dev/null +++ b/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +#SBATCH --job-name="iconsub_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --chdir={cfg.icon_work} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --nodes=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --output={logfile} +#SBATCH --open-mode=append + +ulimit -s unlimited + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh +spack load icontools + +#----------------------------------------------------------------------------- +# PART I: Create auxiliary grid file which contains only the cells of the +# boundary zone. +#----------------------------------------------------------------------------- +cat > NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} << EOF_1 +&iconsub_nml + grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}', + output_type = 4, + lwrite_grid = .TRUE., +/ +&subarea_nml + ORDER = "{cfg.lateral_boundary_grid_order}", + grf_info_file = '{cfg.input_files_scratch_dynamics_grid_filename}', + min_refin_c_ctrl = 1 + max_refin_c_ctrl = 14 +/ +EOF_1 + +srun -n 1 {cfg.iconsub_bin} \ + --nml NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} 2>&1 + +#----------------------------------------------------------------------------- +# PART II: Extract boundary data +#----------------------------------------------------------------------------- +rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* + +set +x +cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF +! +&input_field_nml ! temperature + inputname = "T" + outputname = "T" + code = 130 + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "U" + outputname = "U" + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "V" + outputname = "V" + intp_method = 3 +/ +&input_field_nml ! vertical velocity + inputname = "OMEGA" + outputname = "W" + code = 135 + intp_method = 3 +/ +&input_field_nml ! surface pressure + inputname = "LNSP" + outputname = "LNPS" + code = 152 + intp_method = 3 +/ +&input_field_nml ! geopotential + inputname = "Z" + outputname = "GEOSP" + code = 129 + intp_method = 3 +/ +&input_field_nml ! specific humidity + inputname = "QV" + outputname = "QV" + code = 133 + intp_method = 3 +/ +&input_field_nml ! cloud liquid water content + inputname = "CLWC" + outputname = "QC" + code = 246 + intp_method = 3 +/ +&input_field_nml ! cloud ice water content + inputname = "CIWC" + outputname = "QI" + code = 247 + intp_method = 3 +/ +&input_field_nml ! rain water content + inputname = "CRWC" + outputname = "QR" + code = 75 + intp_method = 3 +/ +&input_field_nml ! snow water content + inputname = "CSWC" + outputname = "QS" + code = 76 + intp_method = 3 +/ +EOF + +#----------------------------------------------------------------------------- +# loop over file list: + +echo "DATAFILELIST is {datafile_list}" +for datafilename in {datafile_list} ; do + datafile="${{datafilename##*/}}" # get filename without path + outdatafile=${{datafile%.*}} # get filename without suffix + cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C +&remap_nml + in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' + in_filename = '{cfg.meteo_dir}/${{datafile}}' + in_type = 1 + out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' + out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' + out_type = 2 + out_filetype = 4 + l_have3dbuffer = .false. +! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" +/ +EOF_2C + + srun -n 1 {cfg.iconremap_bin} -q \ + --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ + --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 + +done + +#----------------------------------------------------------------------------- +# clean-up + +rm -f nml.log +rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} + +#----------------------------------------------------------------------------- +exit +#----------------------------------------------------------------------------- + diff --git a/cases/icon-async-test/icontools_remap_ic_runjob.cfg b/cases/icon-async-test/icontools_remap_ic_runjob.cfg new file mode 100755 index 00000000..e31b9eb5 --- /dev/null +++ b/cases/icon-async-test/icontools_remap_ic_runjob.cfg @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +#SBATCH --job-name="iconremap_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --chdir={cfg.icon_work} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --nodes=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --output={logfile} +#SBATCH --open-mode=append + +ulimit -s unlimited + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh +spack load icontools + +#----------------------------------------------------------------------------- +# Remap inital data onto local (limited-area) grid +#----------------------------------------------------------------------------- +cat > NAMELIST_ICONREMAP_FIELDS << EOF +! +&input_field_nml ! temperature + inputname = "T" + outputname = "T" + code = 130 + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "U" + outputname = "U" + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "V" + outputname = "V" + intp_method = 3 +/ +&input_field_nml ! vertical velocity + inputname = "OMEGA" + outputname = "W" + code = 135 + intp_method = 3 +/ +&input_field_nml ! surface pressure + inputname = "LNSP" + outputname = "LNPS" + code = 152 + intp_method = 3 +/ +&input_field_nml ! geopotential + inputname = "Z" + outputname = "GEOSP" + code = 129 + intp_method = 3 +/ +&input_field_nml ! geopotential + inputname = "Z" + outputname = "GEOP_SFC" + code = 129 + intp_method = 3 +/ +&input_field_nml ! specific humidity + inputname = "QV" + outputname = "QV" + code = 133 + intp_method = 3 +/ +&input_field_nml ! cloud liquid water content + inputname = "CLWC" + outputname = "QC" + code = 246 + intp_method = 3 +/ +&input_field_nml ! cloud ice water content + inputname = "CIWC" + outputname = "QI" + code = 247 + intp_method = 3 +/ +&input_field_nml ! rain water content + inputname = "CRWC" + outputname = "QR" + code = 75 + intp_method = 3 +/ +&input_field_nml ! snow water content + inputname = "CSWC" + outputname = "QS" + code = 76 + intp_method = 3 +/ +&input_field_nml ! snow temperature + inputname = "TSN" + outputname = "T_SNOW" + code = 238 + intp_method = 3 +/ +&input_field_nml ! water content of snow + inputname = "SD" + outputname = "W_SNOW" + code = 141 + intp_method = 3 +/ +&input_field_nml ! density of snow + inputname = "RSN" + outputname = "RHO_SNOW" + code = 33 + intp_method = 3 +/ +&input_field_nml ! snow albedo + inputname = "ASN" + outputname = "ALB_SNOW" + code = 32 + intp_method = 3 +/ +&input_field_nml ! skin temperature + inputname = "SKT" + outputname = "SKT" + code = 235 + intp_method = 3 +/ +&input_field_nml ! sea surface temperature + inputname = "SST" + outputname = "SST" + code = 34 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 1 + inputname = "STL1" + outputname = "STL1" + code = 139 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 2 + inputname = "STL2" + outputname = "STL2" + code = 170 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 3 + inputname = "STL3" + outputname = "STL3" + code = 183 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 4 + inputname = "STL4" + outputname = "STL4" + code = 236 + intp_method = 3 +/ +&input_field_nml ! sea-ice cover + inputname = "CI" + outputname = "CI" + code = 31 + intp_method = 3 +/ +&input_field_nml ! water cont. of interception storage + inputname = "SRC" + outputname = "W_I" + code = 198 + intp_method = 3 +/ +&input_field_nml ! surface roughness + inputname = "SR" + outputname = "Z0" + code = 173 + intp_method = 3 +/ +&input_field_nml ! Land/sea mask + inputname = "LSM" + outputname = "LSM" + code = 172 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 1 + inputname = "SWVL1" + outputname = "SMIL1" + code = 80 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 2 + inputname = "SWVL2" + outputname = "SMIL2" + code = 81 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 3 + inputname = "SWVL3" + outputname = "SMIL3" + code = 82 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 4 + inputname = "SWVL4" + outputname = "SMIL4" + code = 83 + intp_method = 3 +/ +EOF + +#----------------------------------------------------------------------------- +# loop over file list: + +datafilename={cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix} +datafile="${{datafilename##*/}}" # get filename without path +outdatafile=${{datafile%.*}} # get filename without suffix + +# create ICON master namelist +# ------------------------ +# For a complete list see Namelist_overview and Namelist_overview.pdf + +cat > NAMELIST_ICONREMAP << EOF +&remap_nml + in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' + in_filename = '{cfg.meteo_dir}/${{datafile}}' + in_type = 1 + out_grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}' + out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}.nc' + out_type = 2 + out_filetype = 4 + l_have3dbuffer = .false. +/ +EOF + +srun -n 1 {cfg.iconremap_bin} \ + -vvvvv -q --remap_nml NAMELIST_ICONREMAP \ + --input_field_nml NAMELIST_ICONREMAP_FIELDS 2>&1 + + +#----------------------------------------------------------------------------- +# clean-up + +rm -f ncstorage.tmp* +rm -f nml.log NAMELIST_SUB NAMELIST_ICONREMAP NAMELIST_ICONREMAP_FIELDS + +#----------------------------------------------------------------------------- +exit +#----------------------------------------------------------------------------- diff --git a/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg new file mode 100755 index 00000000..b0950efa --- /dev/null +++ b/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +#SBATCH --job-name="iconremap_lbc_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --chdir={cfg.icon_work} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --nodes=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --output={logfile} +#SBATCH --open-mode=append + +ulimit -s unlimited + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh +spack load icontools + +#----------------------------------------------------------------------------- +# Extract boundary data +#----------------------------------------------------------------------------- + +rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* + +cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF +! +&input_field_nml ! temperature + inputname = "T" + outputname = "T" + code = 130 + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "U" + outputname = "U" + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. v + inputname = "V" + outputname = "V" + intp_method = 3 +/ +&input_field_nml ! vertical velocity + inputname = "OMEGA" + outputname = "W" + code = 135 + intp_method = 3 +/ +&input_field_nml ! surface pressure + inputname = "LNSP" + outputname = "LNPS" + code = 152 + intp_method = 3 +/ +&input_field_nml ! specific humidity + inputname = "QV" + outputname = "QV" + code = 133 + intp_method = 3 +/ +&input_field_nml ! cloud liquid water content + inputname = "CLWC" + outputname = "QC" + code = 246 + intp_method = 3 +/ +&input_field_nml ! cloud ice water content + inputname = "CIWC" + outputname = "QI" + code = 247 + intp_method = 3 +/ +&input_field_nml ! rain water content + inputname = "CRWC" + outputname = "QR" + code = 75 + intp_method = 3 +/ +&input_field_nml ! snow water content + inputname = "CSWC" + outputname = "QS" + code = 76 + intp_method = 3 +/ +EOF + +#----------------------------------------------------------------------------- +# loop over file list: + +echo "DATAFILELIST is {datafile_list_rest}" +for datafilename in {datafile_list_rest} ; do + datafile="${{datafilename##*/}}" # get filename without path + outdatafile=${{datafile%.*}} # get filename without suffix + cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C +&remap_nml + in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' + in_filename = '{cfg.meteo_dir}/${{datafile}}' + in_type = 1 + out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' + out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' + out_type = 2 + out_filetype = 4 + l_have3dbuffer = .false. +! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" +/ +EOF_2C + + srun -n 1 {cfg.iconremap_bin} -q \ + --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ + --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 + +done + +#----------------------------------------------------------------------------- +# clean-up + +#rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* +rm -f nml.log +rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} + +#----------------------------------------------------------------------------- +exit +#----------------------------------------------------------------------------- +