enable MemorySpace in GMG transfer

tjhei · Nov 14, 2024 · 354b270 · 354b270
1 parent 9e9a721
commit 354b270
Show file tree

Hide file tree

Showing 9 changed files with 289 additions and 214 deletions.
diff --git a/cmake/config/template-arguments.in b/cmake/config/template-arguments.in
@@ -169,18 +169,13 @@ EXTERNAL_PARALLEL_VECTORS := { @DEAL_II_EXPAND_TRILINOS_MPI_VECTOR@;
                                @DEAL_II_EXPAND_PETSC_MPI_BLOCKVECTOR@
                              }
 
-// TODO: I don't understand this one. LA::distributed::Vector does not have a
-// native matrix type, especially if we don't compile with Trilinos. Currently
-// only used: mg_transfer_prebuilt.inst.in and
-// mg_level_global_transfer.inst.in
-VECTORS_WITH_MATRIX := { Vector<double>;
+// Vectors we can do GMG with excluding the LinearAlgebra::distributed::Vector<> (which is handled separately):
+VECTORS_WITHOUT_LAVEC := { Vector<double>;
                          Vector<float> ;
 
                          BlockVector<double>;
                          BlockVector<float>;
 
-                         LinearAlgebra::distributed::Vector<double>;
-
                          @DEAL_II_EXPAND_TRILINOS_MPI_VECTOR@;
                          @DEAL_II_EXPAND_EPETRA_VECTOR@;
                          @DEAL_II_EXPAND_TPETRA_VECTOR_DOUBLE@;

diff --git a/include/deal.II/multigrid/mg_transfer.h b/include/deal.II/multigrid/mg_transfer.h
@@ -407,9 +407,11 @@ class MGLevelGlobalTransfer : public MGTransferBase<VectorType>
  * routines as compared to the %parallel vectors in the PETScWrappers and
  * TrilinosWrappers namespaces.
  */
-template <typename Number>
-class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
-  : public MGTransferBase<LinearAlgebra::distributed::Vector<Number>>
+template <typename Number, typename MemorySpace>
+class MGLevelGlobalTransfer<
+  LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+  : public MGTransferBase<
+      LinearAlgebra::distributed::Vector<Number, MemorySpace>>
 {
 public:
   /**
@@ -426,9 +428,10 @@ class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
    */
   template <int dim, typename Number2, int spacedim>
   void
-  copy_to_mg(const DoFHandler<dim, spacedim> &dof_handler,
-             MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-             const LinearAlgebra::distributed::Vector<Number2> &src) const;
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace> &src) const;
 
   /**
    * Transfer from multi-level vector to normal vector.
@@ -440,9 +443,10 @@ class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
   template <int dim, typename Number2, int spacedim>
   void
   copy_from_mg(
-    const DoFHandler<dim, spacedim>             &dof_handler,
-    LinearAlgebra::distributed::Vector<Number2> &dst,
-    const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const;
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const;
 
   /**
    * Add a multi-level vector to a normal vector.
@@ -452,9 +456,10 @@ class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
   template <int dim, typename Number2, int spacedim>
   void
   copy_from_mg_add(
-    const DoFHandler<dim, spacedim>             &dof_handler,
-    LinearAlgebra::distributed::Vector<Number2> &dst,
-    const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const;
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const;
 
   /**
    * If this object operates on BlockVector objects, we need to describe how
@@ -493,10 +498,11 @@ class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
    */
   template <int dim, typename Number2, int spacedim>
   void
-  copy_to_mg(const DoFHandler<dim, spacedim> &dof_handler,
-             MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-             const LinearAlgebra::distributed::Vector<Number2>         &src,
-             const bool solution_transfer) const;
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace>         &src,
+    const bool solution_transfer) const;
 
   /**
    * Internal function to @p fill copy_indices*. Called by derived classes.
@@ -581,26 +587,27 @@ class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
    * global vector for inserting into the level vectors. This vector is
    * populated with those entries.
    */
-  mutable LinearAlgebra::distributed::Vector<Number> ghosted_global_vector;
+  mutable LinearAlgebra::distributed::Vector<Number, MemorySpace>
+    ghosted_global_vector;
 
   /**
    * Same as above but used when working with solution vectors.
    */
-  mutable LinearAlgebra::distributed::Vector<Number>
+  mutable LinearAlgebra::distributed::Vector<Number, MemorySpace>
     solution_ghosted_global_vector;
 
   /**
    * In the function copy_from_mg, we access all level vectors with certain
    * ghost entries for inserting the result into a global vector.
    */
-  mutable MGLevelObject<LinearAlgebra::distributed::Vector<Number>>
+  mutable MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
     ghosted_level_vector;
 
   /**
    * Function to initialize internal level vectors.
    */
   std::function<void(const unsigned int,
-                     LinearAlgebra::distributed::Vector<Number> &)>
+                     LinearAlgebra::distributed::Vector<Number, MemorySpace> &)>
     initialize_dof_vector;
 
 private:

diff --git a/include/deal.II/multigrid/mg_transfer.templates.h b/include/deal.II/multigrid/mg_transfer.templates.h
@@ -394,27 +394,29 @@ MGLevelGlobalTransfer<VectorType>::assert_built(
 /* --------- MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector> -------
  */
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_to_mg(
-  const DoFHandler<dim, spacedim>                           &dof_handler,
-  MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-  const LinearAlgebra::distributed::Vector<Number2>         &src) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace> &src) const
 {
   assert_built(dof_handler);
   copy_to_mg(dof_handler, dst, src, false);
 }
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_to_mg(
-  const DoFHandler<dim, spacedim>                           &dof_handler,
-  MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-  const LinearAlgebra::distributed::Vector<Number2>         &src,
-  const bool solution_transfer) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace>         &src,
+    const bool solution_transfer) const
 {
   assert_built(dof_handler);
   LinearAlgebra::distributed::Vector<Number> &this_ghosted_global_vector =
@@ -513,13 +515,15 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_to_mg(
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_from_mg(
-  const DoFHandler<dim, spacedim>                                 &dof_handler,
-  LinearAlgebra::distributed::Vector<Number2>                     &dst,
-  const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  copy_from_mg(
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const
 {
   assert_built(dof_handler);
   AssertIndexRange(src.max_level(),
@@ -598,14 +602,15 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_from_mg(
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
   copy_from_mg_add(
-    const DoFHandler<dim, spacedim>             &dof_handler,
-    LinearAlgebra::distributed::Vector<Number2> &dst,
-    const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const
 {
   assert_built(dof_handler);
   // For non-DG: degrees of freedom in the refinement face may need special
@@ -645,19 +650,19 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
   set_component_to_block_map(const std::vector<unsigned int> &map)
 {
   component_to_block_map = map;
 }
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::assert_built(
-  const DoFHandler<dim, spacedim> &dof_handler) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  assert_built(const DoFHandler<dim, spacedim> &dof_handler) const
 {
   (void)dof_handler;
   Assert(copy_indices.size() ==