ode_norm, ode_unstable_check

src/Trixi.jl

@@ -53,7 +53,7 @@ using RecipesBase: RecipesBase
 using Requires: @require
 using Static: Static, One
 @reexport using StaticArrays: SVector
-using StaticArrays: MVector, MArray, SMatrix, @SMatrix
+using StaticArrays: StaticArrays, MVector, MArray, SMatrix, @SMatrix
 using StrideArrays: PtrArray, StrideArray, StaticInt
 @reexport using StructArrays: StructArrays, StructArray
 using TimerOutputs: TimerOutputs, @notimeit, TimerOutput, print_timer, reset_timer!
@@ -203,7 +203,10 @@ export ControllerThreeLevel, ControllerThreeLevelCombined,
 
 export PositivityPreservingLimiterZhangShu
 
-export trixi_include, examples_dir, get_examples, default_example, default_example_unstructured
+export trixi_include, examples_dir, get_examples, default_example,
+       default_example_unstructured
+
+export ode_norm, ode_unstable_check
 
 export convergence_test, jacobian_fd, jacobian_ad_forward, linear_structure
 

src/auxiliary/mpi.jl

@@ -1,9 +1,3 @@
-# By default, Julia/LLVM does not use fused multiply-add operations (FMAs).
-# Since these FMAs can increase the performance of many numerical algorithms,
-# we need to opt-in explicitly.
-# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details.
-@muladd begin
-
 
 """
     init_mpi()
@@ -75,4 +69,67 @@ end
 end
 
 
-end # @muladd
+"""
+    ode_norm(u, t)
+
+Implementation of the weighted L2 norm of Hairer and Wanner used for error-based
+step size control in OrdinaryDiffEq.jl. This function is aware of MPI and uses
+global MPI communication when running in parallel.
+
+You must pass this function as a keyword argument
+`internalnorm=ode_norm`
+to OrdinaryDiffEq.jl's `solve` when using error-based step size control with MPI
+parallel execution of Trixi.jl.
+
+See [`https://diffeq.sciml.ai/stable/basics/common_solver_opts/#advanced_adaptive_stepsize_control`](https://diffeq.sciml.ai/stable/basics/common_solver_opts/#advanced_adaptive_stepsize_control)
+
+!!! warning "Experimental code"
+    This code is experimental and may be changed or removed in any future release.
+"""
+ode_norm(u::Number, t) = @fastmath abs(u)
+function ode_norm(u::AbstractArray, t)
+  local_sumabs2 = recursive_sum_abs2(u) # sum(abs2, u)
+  local_length  = recursive_length(u)   # length(u)
+  if mpi_isparallel()
+    global_sumabs2, global_length = MPI.Allreduce([local_sumabs2, local_length], +, mpi_comm())
+    return sqrt(global_sumabs2 / global_length)
+  else
+    return sqrt(local_sumabs2 / local_length)
+  end
+end
+
+# Recursive `sum(abs2, ...)` and `length(...)` are required when dealing with
+# arrays of arrays, e.g., when using `DGMulti` solvers with an array-of-structs
+# (`Array{SVector}`) or a structure-of-arrays (`StructArray`). We need to take
+# care of these situations when allowing to use `ode_norm` as default norm in
+# OrdinaryDiffEq.jl throughout all applications of Trixi.jl.
+recursive_sum_abs2(u::Number) = abs2(u)
+recursive_sum_abs2(u::AbstractArray) = sum(recursive_sum_abs2, u)
+
+recursive_length(u::Number) = length(u)
+recursive_length(u::AbstractArray{<:Number}) = length(u)
+recursive_length(u::AbstractArray{<:AbstractArray}) = sum(recursive_length, u)
+function recursive_length(u::AbstractArray{<:StaticArrays.StaticArray{S, <:Number}}) where {S}
+  prod(StaticArrays.Size(eltype(u))) * length(u)
+end
+
+
+"""
+    ode_unstable_check(dt, u, semi, t)
+
+Implementation of the basic check for instability used in OrdinaryDiffEq.jl.
+Instead of checking something like `any(isnan, u)`, this function just checks
+`isnan(dt)`. This helps when using MPI parallelization, since no additional
+global communication is required and all ranks will return the same result.
+
+You should pass this function as a keyword argument
+`unstable_check=ode_unstable_check`
+to OrdinaryDiffEq.jl's  `solve` when using error-based step size control with MPI
+parallel execution of Trixi.jl.
+
+See [`https://diffeq.sciml.ai/stable/basics/common_solver_opts/#Miscellaneous`](https://diffeq.sciml.ai/stable/basics/common_solver_opts/#Miscellaneous)
+
+!!! warning "Experimental code"
+    This code is experimental and may be changed or removed in any future release.
+"""
+ode_unstable_check(dt, u, semi, t) = isnan(dt)

test/test_mpi.jl

@@ -26,6 +26,22 @@ CI_ON_WINDOWS = (get(ENV, "GITHUB_ACTIONS", false) == "true") && Sys.iswindows()
   end
 end # MPI
 
+
+@trixi_testset "MPI supporting functionality" begin
+  using OrdinaryDiffEq
+
+  t = 0.5
+  let u = 1.0
+    @test ode_norm(u, t) ≈ OrdinaryDiffEq.ODE_DEFAULT_NORM(u, t)
+  end
+  let u = [1.0, -2.0]
+    @test ode_norm(u, t) ≈ OrdinaryDiffEq.ODE_DEFAULT_NORM(u, t)
+  end
+  let u = [SVector(1.0, -2.0), SVector(0.5, -0.1)]
+    @test ode_norm(u, t) ≈ OrdinaryDiffEq.ODE_DEFAULT_NORM(u, t)
+  end
+end # MPI supporting functionality
+
 # Clean up afterwards: delete Trixi output directory
 Trixi.mpi_isroot() && @test_nowarn rm(outdir, recursive=true)
 

test/test_mpi_p4est_2d.jl

@@ -18,6 +18,21 @@ const EXAMPLES_DIR = joinpath(pathof(Trixi) |> dirname |> dirname, "examples", "
       # Expected errors are exactly the same as with TreeMesh!
       l2   = [8.311947673061856e-6],
       linf = [6.627000273229378e-5])
+
+    @testset "error-based step size control" begin
+      Trixi.mpi_isroot() && println("-"^100)
+      Trixi.mpi_isroot() && println("elixir_advection_basic.jl with error-based step size control")
+
+      sol = solve(ode, RDPK3SpFSAL35(), abstol=1.0e-4, reltol=1.0e-4,
+                  save_everystep=false, callback=callbacks,
+                  internalnorm=ode_norm,
+                  unstable_check=ode_unstable_check); summary_callback()
+      errors = analysis_callback(sol)
+      if Trixi.mpi_isroot()
+        @test errors.l2 ≈ [3.3022040342579066e-5]    rtol=1.0e-4
+        @test errors.linf ≈ [0.00011787417954578494] rtol=1.0e-4
+      end
+    end
   end
 
   @trixi_testset "elixir_advection_nonconforming_flag.jl" begin

test/test_mpi_p4est_3d.jl

@@ -18,6 +18,21 @@ const EXAMPLES_DIR = joinpath(pathof(Trixi) |> dirname |> dirname, "examples", "
       # Expected errors are exactly the same as with TreeMesh!
       l2   = [0.00016263963870641478],
       linf = [0.0014537194925779984])
+
+    @testset "error-based step size control" begin
+      Trixi.mpi_isroot() && println("-"^100)
+      Trixi.mpi_isroot() && println("elixir_advection_basic.jl with error-based step size control")
+
+      sol = solve(ode, RDPK3SpFSAL35(), abstol=1.0e-4, reltol=1.0e-4,
+                  save_everystep=false, callback=callbacks,
+                  internalnorm=ode_norm,
+                  unstable_check=ode_unstable_check); summary_callback()
+      errors = analysis_callback(sol)
+      if Trixi.mpi_isroot()
+        @test errors.l2 ≈ [0.00016800412839949264]  rtol=1.0e-4
+        @test errors.linf ≈ [0.0014548839020096516] rtol=1.0e-4
+      end
+    end
   end
 
   @trixi_testset "elixir_advection_amr.jl" begin

test/test_mpi_tree.jl

@@ -124,6 +124,21 @@ const EXAMPLES_DIR = joinpath(pathof(Trixi) |> dirname |> dirname, "examples", "
     @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_ec.jl"),
       l2   = [0.061751715597716854, 0.05018223615408711, 0.05018989446443463, 0.225871559730513],
       linf = [0.29347582879608825, 0.31081249232844693, 0.3107380389947736, 1.0540358049885143])
+
+    @testset "error-based step size control" begin
+      Trixi.mpi_isroot() && println("-"^100)
+      Trixi.mpi_isroot() && println("elixir_euler_ec.jl with error-based step size control")
+
+      sol = solve(ode, RDPK3SpFSAL35(), abstol=1.0e-4, reltol=1.0e-4,
+                  save_everystep=false, callback=callbacks,
+                  internalnorm=ode_norm,
+                  unstable_check=ode_unstable_check); summary_callback()
+      errors = analysis_callback(sol)
+      if Trixi.mpi_isroot()
+        @test errors.l2   ≈ [0.061653630426688116, 0.05006930431098764, 0.05007694316484242, 0.22550689872331683] rtol=1.0e-4
+        @test errors.linf ≈ [0.28516937484583693, 0.2983633696512788, 0.297812036335975, 1.027368795517512]       rtol=1.0e-4
+      end
+    end
   end
 
   @trixi_testset "elixir_euler_vortex.jl" begin