moved all needed functions.

src/coreComponents/finiteVolume/TwoPointFluxApproximation.cpp

@@ -31,6 +31,7 @@
 #include "finiteVolume/SurfaceElementStencil.hpp"
 #include "mesh/SurfaceElementRegion.hpp"
 #include "mesh/utilities/ComputationalGeometry.hpp"
+#include "mesh/utilities/CIcomputationKernel.hpp"
 #include "physicsSolvers/fluidFlow/FlowSolverBaseFields.hpp"
 
 #include "LvArray/src/tensorOps.hpp"
@@ -91,6 +92,11 @@ void TwoPointFluxApproximation::computeFractureStencil( MeshLevel & mesh ) const
       string const & regionName = region.getName();
       addFractureFractureConnectionsDFM( mesh, regionName );
       addFractureMatrixConnectionsDFM( mesh, regionName );
+    } else if ( region.subRegionType() == SurfaceElementRegion::SurfaceSubRegionType::embeddedSurfaceElement )
+    {
+      string const & regionName = region.getName();
+      addFractureFractureConnectionsEDFM( mesh, regionName );
+      addFractureMatrixConnectionsEDFM( mesh, regionName );
     }
   } );
 }
@@ -547,6 +553,8 @@ void TwoPointFluxApproximation::addFractureMatrixConnectionsEDFM( MeshLevel & me
 
   EmbeddedSurfaceSubRegion & fractureSubRegion = fractureRegion.getUniqueSubRegion< EmbeddedSurfaceSubRegion >();
 
+  computeConnectivityIndex( elemManager, mesh.getNodeManager();, embeddedSurfaceSubRegion );
+
   ElementRegionManager::ElementViewAccessor< arrayView1d< real64 const > > hydraulicAperture =
     elemManager.constructViewAccessor< array1d< real64 >, arrayView1d< real64 const > >( fields::flow::hydraulicAperture::key() );
 
@@ -605,6 +613,23 @@ void TwoPointFluxApproximation::addFractureMatrixConnectionsEDFM( MeshLevel & me
   }
 
 }
+void TwoPointFluxApproximation::computeConnectivityIndex( ElementRegionManager & elemManager, 
+                                                          NodeManager & nodeManager, 
+                                                          EmbeddedSurfaceSubRegion & embeddedSurfaceSubRegion ) const
+{
+  // Launch kernel to compute connectivity index of each fractured element.
+  elemManager.forElementSubRegionsComplete< CellElementSubRegion >(
+    [&]( localIndex const, localIndex const, ElementRegionBase &, CellElementSubRegion & subRegion )
+  {
+    auto kernelVariant = createKernel( subRegion.getElementType(), nodeManager, subRegion, embeddedSurfaceSubRegion ); 
+
+    std::visit( [&] ( auto & kernel ) 
+    {
+      using KERNEL_TYPE = decltype( kernel );
+      KERNEL_TYPE::template launchCIComputationKernel< parallelDevicePolicy< >, KERNEL_TYPE >( kernel );
+    }, kernelVariant);  
+  } );
+}
 
 void TwoPointFluxApproximation::addFractureFractureConnectionsEDFM( MeshLevel & mesh,
                                                                     string const & embeddedSurfaceRegionName ) const

src/coreComponents/finiteVolume/TwoPointFluxApproximation.hpp

@@ -145,6 +145,17 @@ class TwoPointFluxApproximation : public FluxApproximationBase
    */
   void initNewFractureFieldsDFM( MeshLevel & mesh,
                                  string const & faceElementRegionName ) const;
+
+  /**
+   * @brief Compute connectivity index.
+   * 
+   * @param elemManager The elementRegionManager
+   * @param nodeManager The nodeManager
+   * @param embeddedSurfaceSubRegion The embeddeSurfaceSubRegion
+   */
+  void computeConnectivityIndex( ElementRegionManager & elemManager, 
+                                 NodeManager & nodeManager, 
+                                 EmbeddedSurfaceSubRegion & embeddedSurfaceSubRegion ) const;                               
 
   /// mean permeability coefficient
   real64 m_meanPermCoefficient;

src/coreComponents/mainInterface/ProblemManager.cpp

@@ -686,7 +686,7 @@ void ProblemManager::generateMesh()
   domain.outputPartitionInformation();
 
   // Create Embedded fractures here.
-  generateEmbeddedFractures();
+  generateEmbeddedSurfacesMesh();
 
   domain.forMeshBodies( [&]( MeshBody & meshBody )
   {
@@ -731,7 +731,7 @@ void ProblemManager::generateMesh()
 
 }
 
-void ProblemManager::generateEmbeddedFractures() const
+void ProblemManager::generateEmbeddedSurfacesMesh() const
 {
   DomainPartition & domain = getDomainPartition();
 
@@ -740,8 +740,19 @@ void ProblemManager::generateEmbeddedFractures() const
   Group const & embSurfBlocks = cellBlockManager.getEmbeddedSurfaceBlocks();
   string const & faceBlockName = embeddedSurfaceRegion.getFaceBlockName();
 
+  // Get meshLevel
+  MeshLevel & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization();
   if( embSurfBlocks.hasGroup( faceBlockName ))
   {
+    // Get managers
+    ElementRegionManager & elemManager = meshLevel.getElemManager();
+    NodeManager & nodeManager = meshLevel.getNodeManager();
+    EmbeddedSurfaceNodeManager & embSurfNodeManager = meshLevel.getEmbSurfNodeManager();
+    EdgeManager & edgeManager = meshLevel.getEdgeManager();
+    arrayView2d< real64 const, nodes::REFERENCE_POSITION_USD > const & nodesCoord = nodeManager.referencePosition();
+
+    NewObjectLists newObjects;
+
     EmbeddedSurfaceBlockABC const & embSurf = embSurfBlocks.getGroup< EmbeddedSurfaceBlockABC >( faceBlockName );
 
     elemManager.forElementSubRegionsComplete< CellElementSubRegion >( [&]( localIndex const er,
@@ -760,13 +771,52 @@ void ProblemManager::generateEmbeddedFractures() const
                                                              embSurf );
 
       // Add all the fracture information to the CellElementSubRegion
-      for( localIndex edfmIndex=0; edfmIndex < embSurf.numEmbeddedSurfElem(); ++edfmIndex )
+      for( localIndex edfmIndex = 0; edfmIndex < embSurf.numEmbeddedSurfElem(); ++edfmIndex )
       {
         localIndex cellIndex = embSurf.getEmbeddedSurfElemTo3dElem().toCellIndex[edfmIndex][0];
         subRegion.addFracturedElement( cellIndex, edfmIndex );
         newObjects.newElements[ {embeddedSurfaceRegion.getIndexInParent(), embeddedSurfaceSubRegion.getIndexInParent()} ].insert( edfmIndex );
       }
     } );   // end loop over subregions
+
+    // add all new nodes to newObject list
+    for( localIndex ni = 0; ni < embSurfNodeManager.size(); ni++ )
+    {
+      newObjects.newNodes.insert( ni );
+    }
+
+    // Set the ghostRank form the parent cell
+    ElementRegionManager::ElementViewAccessor< arrayView1d< integer const > > const & cellElemGhostRank =
+      elemManager.constructArrayViewAccessor< integer, 1 >( ObjectManagerBase::viewKeyStruct::ghostRankString() );
+
+    embeddedSurfaceSubRegion.inheritGhostRank( cellElemGhostRank );
+
+    setGlobalIndices( elemManager, embSurfNodeManager, embeddedSurfaceSubRegion );
+
+    embeddedSurfacesParallelSynchronization::sychronizeTopology( meshLevel,
+                                                                 domain.getNeighbors(),
+                                                                 newObjects,
+                                                                 m_mpiCommOrder,
+                                                                 this->m_fractureRegionName );
+
+    addEmbeddedElementsToSets( elemManager, embeddedSurfaceSubRegion );
+
+    EmbeddedSurfaceSubRegion::NodeMapType & embSurfToNodeMap = embeddedSurfaceSubRegion.nodeList();
+
+    // Populate EdgeManager for embedded surfaces.
+    EdgeManager & embSurfEdgeManager = meshLevel.getEmbSurfEdgeManager();
+
+    EmbeddedSurfaceSubRegion::EdgeMapType & embSurfToEdgeMap = embeddedSurfaceSubRegion.edgeList();
+
+    localIndex numOfPoints = embSurfNodeManager.size();
+
+    // Create the edges
+    embSurfEdgeManager.buildEdges( numOfPoints, embSurfToNodeMap.toViewConst(), embSurfToEdgeMap );
+    // Node to cell map
+    embSurfNodeManager.setElementMaps( elemManager );
+    // Node to edge map
+    embSurfNodeManager.setEdgeMaps( embSurfEdgeManager );
+    embSurfNodeManager.compressRelationMaps();
   }
 }
 

src/coreComponents/mesh/CMakeLists.txt

@@ -38,6 +38,7 @@ set( mesh_headers
      ExternalDataSourceBase.hpp
      ExternalDataSourceManager.hpp
      EmbeddedSurfaceNodeManager.hpp
+     EmbeddedSurfacesParallelSynchronization.hpp
      EmbeddedSurfaceSubRegion.hpp
      MeshFields.hpp
      FaceElementSubRegion.hpp
@@ -128,6 +129,7 @@ set( mesh_sources
      ElementRegionManager.cpp
      ElementSubRegionBase.cpp
      EmbeddedSurfaceNodeManager.cpp
+     EmbeddedSurfacesParallelSynchronization.cpp
      EmbeddedSurfaceSubRegion.cpp
      ExternalDataSourceBase.cpp
      ExternalDataSourceManager.cpp

src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfacesParallelSynchronization.hpp → src/coreComponents/mesh/EmbeddedSurfacesParallelSynchronization.hpp

@@ -21,10 +21,19 @@
 #ifndef GEOS_PHYSICSSOLVERS_SURFACEGENERATION_EMBEDDEDSURFACESPARALLELSYNCHRONIZATION_HPP_
 #define GEOS_PHYSICSSOLVERS_SURFACEGENERATION_EMBEDDEDSURFACESPARALLELSYNCHRONIZATION_HPP_
 
-#include "physicsSolvers/surfaceGeneration/EmbeddedSurfaceGenerator.hpp"
 
 namespace geos
 {
+
+struct NewObjectLists
+{
+  std::set< localIndex > newNodes;
+  std::set< localIndex > newEdges;
+  map< std::pair< localIndex, localIndex >, std::set< localIndex > > newElements;
+
+  void insert( NewObjectLists const & lists );
+};
+
 class MeshLevel;
 class NeighborCommunicator;
 struct ModifiedObjectLists;

src/coreComponents/mesh/generators/VTKUtilities.cpp

@@ -697,8 +697,11 @@ AllMeshes redistributeByCellGraph( AllMeshes & input,
   }
 
   // Ouassim: just add the edfm mesh at the moment and see.
-  auto edfmMesh = input.getEmbeddedSurfaceBlocks();
-  return AllMeshes( finalMesh, finalFractures, edfmMesh );
+  // Create edfmMeshPartitions using newPartitions and the map between the global cell ids and the global edfm fracture cell ids
+  vtkSmartPointer< vtkPartitionedDataSet > const edfmSplitMesh = splitMeshByPartition( input.getEmbeddedSurfaceBlocks(), numRanks, edfmMeshPartitions.toViewConst() );
+  vtkSmartPointer< vtkUnstructuredGrid > finalEDFMMesh = vtk::redistribute( *edfmSplitMesh, MPI_COMM_GEOS );
+  // auto edfmMesh = input.getEmbeddedSurfaceBlocks();
+  return AllMeshes( finalMesh, finalFractures, finalEDFMMesh );
 }
 
 /**

src/coreComponents/mesh/utilities/CIcomputationKernel.hpp

@@ -44,17 +44,13 @@ class CIcomputationKernel
   /**
    * @brief Construct a new CIcomputationKernel object
    *
-   * @param finiteElementSpace the finite element space
    * @param nodeManager the nodeManager
    * @param elementSubRegion the element subRegion
    * @param embeddedSurfSubRegion the embeddedSurfaceSubRegion
    */
-  CIcomputationKernel( FE_TYPE const & finiteElementSpace,
-                       NodeManager const & nodeManager,
+  CIcomputationKernel( NodeManager const & nodeManager,
                        CellElementSubRegion const & elementSubRegion,
                        EmbeddedSurfaceSubRegion & embeddedSurfSubRegion ):
-    m_finiteElementSpace( finiteElementSpace ),
-    m_elementType( elementSubRegion.getElementType() ),
     m_X( nodeManager.referencePosition() ),
     m_elemsToNodes( elementSubRegion.nodeList().toViewConst() ),
     m_fracturedElems( elementSubRegion.fracturedElementsList().toViewConst()),
@@ -211,12 +207,6 @@ class CIcomputationKernel
 
 private:
 
-  /// the finite element space
-  FE_TYPE const & m_finiteElementSpace;
-
-  /// the element type
-  ElementType const m_elementType;
-
   /// the reference position of the nodes
   arrayView2d< real64 const, nodes::REFERENCE_POSITION_USD > const m_X;
 
@@ -242,6 +232,32 @@ class CIcomputationKernel
   arrayView1d< real64 > const m_connectivityIndex;
 };
 
+using KernelVariant = std::variant< CIcomputationKernel< H1_Hexahedron_Lagrange1_GaussLegendre2  >,
+                                    CIcomputationKernel< H1_Wedge_Lagrange1_Gauss6 >,
+                                    CIcomputationKernel< H1_Tetrahedron_Lagrange1_Gauss1 >,
+                                    CIcomputationKernel< H1_Pyramid_Lagrange1_Gauss5 > >;
+
+KernelVariant createKernel( ElementType elemType, 
+                            NodeManager const & nodeManager,
+                            CellElementSubRegion const & subRegion,
+                            EmbeddedSurfaceSubRegion & esr )
+{
+    switch ( elemType ) 
+    {
+      case ElementType::Tetrahedron:
+        return CIcomputationKernel<H1_Tetrahedron_Lagrange1_Gauss1>( nodeManager, subRegion, esr );
+      case ElementType::Hexahedron:
+        return CIcomputationKernel<H1_Hexahedron_Lagrange1_GaussLegendre2>( nodeManager, subRegion, esr );
+      case ElementType::Pyramid
+        return CIcomputationKernel<H1_Pyramid_Lagrange1_Gauss5>( nodeManager, subRegion, esr );
+      case ElementType::Wedge
+        return CIcomputationKernel<H1_Wedge_VEM_Gauss1>( nodeManager, subRegion, esr );  
+      default:
+        GEOS_ERROR( "Unknown element type" );
+        return {};
+    }
+}
+
 }
 
 #endif /* GEOS_MESH_UTILITIES_CICOMPUTATIONKERNEL_HPP_ */

src/coreComponents/physicsSolvers/surfaceGeneration/CMakeLists.txt

@@ -2,7 +2,6 @@
 set( physicsSolvers_headers
      ${physicsSolvers_headers}
      surfaceGeneration/EmbeddedSurfaceGenerator.hpp
-     surfaceGeneration/EmbeddedSurfacesParallelSynchronization.hpp
      surfaceGeneration/ParallelTopologyChange.hpp
      surfaceGeneration/SurfaceGenerator.hpp
      surfaceGeneration/SurfaceGeneratorFields.hpp
@@ -15,7 +14,6 @@ set( physicsSolvers_headers
 set( physicsSolvers_sources
      ${physicsSolvers_sources}
      surfaceGeneration/EmbeddedSurfaceGenerator.cpp
-     surfaceGeneration/EmbeddedSurfacesParallelSynchronization.cpp
      surfaceGeneration/ParallelTopologyChange.cpp
      surfaceGeneration/ParallelTopologyChangeNoFixup.cpp
      surfaceGeneration/SurfaceGenerator.cpp

src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfaceGenerator.cpp

@@ -18,8 +18,7 @@
  */
 
 #include "EmbeddedSurfaceGenerator.hpp"
-#include "EmbeddedSurfacesParallelSynchronization.hpp"
-
+#include "mesh/EmbeddedSurfacesParallelSynchronization.hpp"
 #include "mesh/mpiCommunications/CommunicationTools.hpp"
 #include "mesh/mpiCommunications/NeighborCommunicator.hpp"
 #include "mesh/mpiCommunications/SpatialPartition.hpp"
@@ -31,7 +30,6 @@
 #include "mesh/SurfaceElementRegion.hpp"
 #include "mesh/MeshFields.hpp"
 #include "mesh/utilities/ComputationalGeometry.hpp"
-#include "mesh/utilities/CIcomputationKernel.hpp"
 #include "physicsSolvers/solidMechanics/kernels/SolidMechanicsLagrangianFEMKernels.hpp"
 #include "mesh/simpleGeometricObjects/GeometricObjectManager.hpp"
 #include "mesh/simpleGeometricObjects/Rectangle.hpp"
@@ -206,27 +204,6 @@ void EmbeddedSurfaceGenerator::initializePostSubGroups()
     } );// end loop over subregions
   } );// end loop over planes
 
-  // Launch kernel to compute connectivity index of each fractured element.
-  elemManager.forElementSubRegionsComplete< CellElementSubRegion >(
-    [&]( localIndex const, localIndex const, ElementRegionBase &, CellElementSubRegion & subRegion )
-  {
-    finiteElement::FiniteElementBase & subRegionFE = subRegion.template getReference< finiteElement::FiniteElementBase >( getDiscretizationName() );
-
-    finiteElement::dispatchlowOrder3D( subRegionFE, [&] ( auto const finiteElement )
-    {
-      using FE_TYPE = decltype( finiteElement );
-
-      auto kernel = CIcomputationKernel< FE_TYPE >( finiteElement,
-                                                    nodeManager,
-                                                    subRegion,
-                                                    embeddedSurfaceSubRegion );
-
-      using KERNEL_TYPE = decltype( kernel );
-
-      KERNEL_TYPE::template launchCIComputationKernel< parallelDevicePolicy< >, KERNEL_TYPE >( kernel );
-    } );
-  } );
-
   // add all new nodes to newObject list
   for( localIndex ni = 0; ni < embSurfNodeManager.size(); ni++ )
   {

src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfaceGenerator.hpp

@@ -28,16 +28,6 @@
 namespace geos
 {
 
-struct NewObjectLists
-{
-  std::set< localIndex > newNodes;
-  std::set< localIndex > newEdges;
-  map< std::pair< localIndex, localIndex >, std::set< localIndex > > newElements;
-
-  void insert( NewObjectLists const & lists );
-};
-
-
 class SpatialPartition;
 
 class NodeManager;