From 6c73dbf0af422da28d8e0b1d418eb59253c5bb8c Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Tue, 15 Oct 2024 11:22:37 -0600 Subject: [PATCH] MueLu RefMaxwell: Add async prolongations Signed-off-by: Christian Glusa --- .../src/Operators/MueLu_RefMaxwell_decl.hpp | 4 +- .../src/Operators/MueLu_RefMaxwell_def.hpp | 312 ++++++++++++------ 2 files changed, 214 insertions(+), 102 deletions(-) diff --git a/packages/muelu/src/Operators/MueLu_RefMaxwell_decl.hpp b/packages/muelu/src/Operators/MueLu_RefMaxwell_decl.hpp index ff143ef7459d..31ad10bbf0a1 100644 --- a/packages/muelu/src/Operators/MueLu_RefMaxwell_decl.hpp +++ b/packages/muelu/src/Operators/MueLu_RefMaxwell_decl.hpp @@ -687,7 +687,7 @@ class RefMaxwell : public VerboseObject, public Xpetra::Operator CoarseNullspace22_; //! Importer to coarse (1,1) hierarchy Teuchos::RCP ImporterCoarse11_, Importer22_; - bool Dk_1_T_R11_colMapsMatch_; + bool Dk_1_T_R11_colMapsMatch_, asyncTransfers_; bool onlyBoundary11_, onlyBoundary22_; //! Parameter lists Teuchos::ParameterList parameterList_, precList11_, precList22_; @@ -700,7 +700,7 @@ class RefMaxwell : public VerboseObject, public Xpetra::Operator P11res_, P11x_, P11resSubComm_, P11xSubComm_, DresIntermediate_, Dres_, DxIntermediate_, Dx_, DresSubComm_, DxSubComm_, residual_, P11resTmp_, DresTmp_, DTR11Tmp_; + mutable Teuchos::RCP P11res_, P11x_, P11resSubComm_, P11xSubComm_, DresIntermediate_, Dres_, DxIntermediate_, Dx_, DresSubComm_, DxSubComm_, residual_, P11resTmp_, DresTmp_, DTR11Tmp_, P11x_colmap_, Dx_colmap_; }; } // namespace MueLu diff --git a/packages/muelu/src/Operators/MueLu_RefMaxwell_def.hpp b/packages/muelu/src/Operators/MueLu_RefMaxwell_def.hpp index b84221b395a3..e109ef5a3c6f 100644 --- a/packages/muelu/src/Operators/MueLu_RefMaxwell_def.hpp +++ b/packages/muelu/src/Operators/MueLu_RefMaxwell_def.hpp @@ -79,18 +79,18 @@ T pop(Teuchos::ParameterList &pl, std::string const &name_in, T def_value) { } template -Teuchos::RCP > -Matrix2CrsMatrix(Teuchos::RCP > &matrix) { - return Teuchos::rcp_dynamic_cast >(matrix, true)->getCrsMatrix(); +Teuchos::RCP> +Matrix2CrsMatrix(Teuchos::RCP> &matrix) { + return Teuchos::rcp_dynamic_cast>(matrix, true)->getCrsMatrix(); } template -const Teuchos::RCP > RefMaxwell::getDomainMap() const { +const Teuchos::RCP> RefMaxwell::getDomainMap() const { return SM_Matrix_->getDomainMap(); } template -const Teuchos::RCP > RefMaxwell::getRangeMap() const { +const Teuchos::RCP> RefMaxwell::getRangeMap() const { return SM_Matrix_->getRangeMap(); } @@ -102,27 +102,27 @@ RefMaxwell:: RCP params = rcp(new ParameterList("RefMaxwell")); - params->set >("Dk_1", Teuchos::null); - params->set >("Dk_2", Teuchos::null); - params->set >("D0", Teuchos::null); + params->set>("Dk_1", Teuchos::null); + params->set>("Dk_2", Teuchos::null); + params->set>("D0", Teuchos::null); - params->set >("M1_beta", Teuchos::null); - params->set >("M1_alpha", Teuchos::null); + params->set>("M1_beta", Teuchos::null); + params->set>("M1_alpha", Teuchos::null); // for backwards compatibility - params->set >("Ms", Teuchos::null); + params->set>("Ms", Teuchos::null); - params->set >("Mk_one", Teuchos::null); - params->set >("Mk_1_one", Teuchos::null); + params->set>("Mk_one", Teuchos::null); + params->set>("Mk_1_one", Teuchos::null); // for backwards compatibility - params->set >("M1", Teuchos::null); + params->set>("M1", Teuchos::null); - params->set >("invMk_1_invBeta", Teuchos::null); - params->set >("invMk_2_invAlpha", Teuchos::null); + params->set>("invMk_1_invBeta", Teuchos::null); + params->set>("invMk_2_invAlpha", Teuchos::null); // for backwards compatibility - params->set >("M0inv", Teuchos::null); + params->set>("M0inv", Teuchos::null); - params->set >("Nullspace", Teuchos::null); - params->set >("Coordinates", Teuchos::null); + params->set>("Nullspace", Teuchos::null); + params->set>("Coordinates", Teuchos::null); auto spaceValidator = rcp(new Teuchos::EnhancedNumberValidator(1, 2)); params->set("refmaxwell: space number", 1, "", spaceValidator); @@ -220,8 +220,8 @@ void RefMaxwell::setParameters(Teucho std::string outputFilename = parameterList_.get("output filename"); if (outputFilename != "") VerboseObject::SetMueLuOFileStream(outputFilename); - if (parameterList_.isType >("output stream")) - VerboseObject::SetMueLuOStream(parameterList_.get >("output stream")); + if (parameterList_.isType>("output stream")) + VerboseObject::SetMueLuOStream(parameterList_.get>("output stream")); if (parameterList_.get("print initial parameters")) GetOStream(static_cast(Runtime1), 0) << parameterList_ << std::endl; @@ -569,6 +569,8 @@ void RefMaxwell::compute(bool reuse) if (Dk_1_T_R11_colMapsMatch_) GetOStream(Runtime0) << solverName_ + "::compute(): Dk_1_T and R11 have matching colMaps" << std::endl; + asyncTransfers_ = (P11_->getRowMap()->lib() == Xpetra::UseTpetra); + // Allocate MultiVectors for solve allocateMemory(1); @@ -696,7 +698,7 @@ void RefMaxwell:: } template -RCP > RefMaxwell:: +RCP> RefMaxwell:: buildAddon(const int spaceNumber) { if (spaceNumber == 0) return Teuchos::null; @@ -958,12 +960,12 @@ void RefMaxwell:: repartFactory->Build(coarseLevel); if (!precList11_.get("repartition: rebalance P and R", false)) - ImporterCoarse11_ = coarseLevel.Get >("Importer", repartFactory.get()); - P11_ = coarseLevel.Get >("P", newP.get()); - coarseA11_ = coarseLevel.Get >("A", newA.get()); - CoordsCoarse11_ = coarseLevel.Get >("Coordinates", newP.get()); + ImporterCoarse11_ = coarseLevel.Get>("Importer", repartFactory.get()); + P11_ = coarseLevel.Get>("P", newP.get()); + coarseA11_ = coarseLevel.Get>("A", newA.get()); + CoordsCoarse11_ = coarseLevel.Get>("Coordinates", newP.get()); if (!NullspaceCoarse11_.is_null()) - NullspaceCoarse11_ = coarseLevel.Get >("Nullspace", newP.get()); + NullspaceCoarse11_ = coarseLevel.Get>("Nullspace", newP.get()); if (!coarseA11_.is_null()) { // Set block size @@ -979,7 +981,7 @@ void RefMaxwell:: if (!disable_addon_ && enable_reuse_) { // Rebalance the addon for next setup - RCP ImporterCoarse11 = coarseLevel.Get >("Importer", repartFactory.get()); + RCP ImporterCoarse11 = coarseLevel.Get>("Importer", repartFactory.get()); RCP targetMap = ImporterCoarse11->getTargetMap(); ParameterList XpetraList; XpetraList.set("Restrict Communicator", true); @@ -1018,11 +1020,11 @@ void RefMaxwell::build22Matrix(const if (!A22_AP_reuse_data_.is_null()) { coarseLevel.AddKeepFlag("AP reuse data", rapFact.get()); - coarseLevel.Set >("AP reuse data", A22_AP_reuse_data_, rapFact.get()); + coarseLevel.Set>("AP reuse data", A22_AP_reuse_data_, rapFact.get()); } if (!A22_RAP_reuse_data_.is_null()) { coarseLevel.AddKeepFlag("RAP reuse data", rapFact.get()); - coarseLevel.Set >("RAP reuse data", A22_RAP_reuse_data_, rapFact.get()); + coarseLevel.Set>("RAP reuse data", A22_RAP_reuse_data_, rapFact.get()); } #ifdef HAVE_MPI @@ -1098,10 +1100,10 @@ void RefMaxwell::build22Matrix(const repartFactory->Build(coarseLevel); if (!precList22_.get("repartition: rebalance P and R", false)) - Importer22_ = coarseLevel.Get >("Importer", repartFactory.get()); - Dk_1_ = coarseLevel.Get >("P", newP.get()); - A22_ = coarseLevel.Get >("A", newA.get()); - Coords22_ = coarseLevel.Get >("Coordinates", newP.get()); + Importer22_ = coarseLevel.Get>("Importer", repartFactory.get()); + Dk_1_ = coarseLevel.Get>("P", newP.get()); + A22_ = coarseLevel.Get>("A", newA.get()); + Coords22_ = coarseLevel.Get>("Coordinates", newP.get()); if (!P22_.is_null()) { // Todo @@ -1116,13 +1118,13 @@ void RefMaxwell::build22Matrix(const coarseLevel.Request("RAP reuse data", rapFact.get()); } - A22_ = coarseLevel.Get >("A", rapFact.get()); + A22_ = coarseLevel.Get>("A", rapFact.get()); if (enable_reuse_) { if (coarseLevel.IsAvailable("AP reuse data", rapFact.get())) - A22_AP_reuse_data_ = coarseLevel.Get >("AP reuse data", rapFact.get()); + A22_AP_reuse_data_ = coarseLevel.Get>("AP reuse data", rapFact.get()); if (coarseLevel.IsAvailable("RAP reuse data", rapFact.get())) - A22_RAP_reuse_data_ = coarseLevel.Get >("RAP reuse data", rapFact.get()); + A22_RAP_reuse_data_ = coarseLevel.Get>("RAP reuse data", rapFact.get()); } } } else { @@ -1221,11 +1223,11 @@ void RefMaxwell::setFineLevelSmoother level.Set("PostSmoother data", PostSmootherData11_, smootherFact.get()); } smootherFact->Build(level); - PreSmoother11_ = level.Get >("PreSmoother", smootherFact.get()); - PostSmoother11_ = level.Get >("PostSmoother", smootherFact.get()); + PreSmoother11_ = level.Get>("PreSmoother", smootherFact.get()); + PostSmoother11_ = level.Get>("PostSmoother", smootherFact.get()); if (enable_reuse_) { - PreSmootherData11_ = level.Get >("PreSmoother data", smootherFact.get()); - PostSmootherData11_ = level.Get >("PostSmoother data", smootherFact.get()); + PreSmootherData11_ = level.Get>("PreSmoother data", smootherFact.get()); + PostSmootherData11_ = level.Get>("PostSmoother data", smootherFact.get()); } } else { std::string smootherType = parameterList_.get("smoother: type"); @@ -1246,10 +1248,10 @@ void RefMaxwell::setFineLevelSmoother level.Set("PreSmoother data", PreSmootherData11_, smootherFact.get()); } smootherFact->Build(level); - PreSmoother11_ = level.Get >("PreSmoother", smootherFact.get()); + PreSmoother11_ = level.Get>("PreSmoother", smootherFact.get()); PostSmoother11_ = PreSmoother11_; if (enable_reuse_) - PreSmootherData11_ = level.Get >("PreSmoother data", smootherFact.get()); + PreSmootherData11_ = level.Get>("PreSmoother data", smootherFact.get()); } } @@ -1318,6 +1320,13 @@ void RefMaxwell::allocateMemory(int n DxSubComm_->setObjectLabel("DxSubComm"); } + if (asyncTransfers_) { + if (!rcp_dynamic_cast(P11_)->getCrsMatrix()->getCrsGraph()->getImporter().is_null()) + P11x_colmap_ = MultiVectorFactory::Build(P11_->getColMap(), numVectors); + if (!rcp_dynamic_cast(Dk_1_)->getCrsMatrix()->getCrsGraph()->getImporter().is_null()) + Dx_colmap_ = MultiVectorFactory::Build(Dk_1_->getColMap(), numVectors); + } + residual_ = MultiVectorFactory::Build(SM_Matrix_->getDomainMap(), numVectors); residual_->setObjectLabel("residual"); } @@ -1371,7 +1380,7 @@ void RefMaxwell::dump(const Kokkos::V } template -Teuchos::RCP RefMaxwell::getTimer(std::string name, RCP > comm) const { +Teuchos::RCP RefMaxwell::getTimer(std::string name, RCP> comm) const { if (IsPrint(Timings)) { if (!syncTimers_) return Teuchos::rcp(new Teuchos::TimeMonitor(*Teuchos::TimeMonitor::getNewTimer("MueLu " + solverName_ + ": " + name))); @@ -1386,7 +1395,7 @@ Teuchos::RCP RefMaxwell -RCP > RefMaxwell:: +RCP> RefMaxwell:: buildNullspace(const int spaceNumber, const Kokkos::View &bcs, const bool applyBCs) { std::string spaceLabel; if (spaceNumber == 0) @@ -1418,7 +1427,7 @@ RCP > RefMaxwell< coordinateType minLen, maxLen, meanLen; if (IsPrint(Statistics2) || normalize) { // compute edge lengths - ArrayRCP > localNullspace(dim_); + ArrayRCP> localNullspace(dim_); for (size_t i = 0; i < dim_; i++) localNullspace[i] = Nullspace->getData(i); coordinateType localMinLen = Teuchos::ScalarTraits::rmax(); @@ -1434,7 +1443,7 @@ RCP > RefMaxwell< localMeanLen += len; } - RCP > comm = Nullspace->getMap()->getComm(); + RCP> comm = Nullspace->getMap()->getComm(); MueLu_minAll(comm, localMinLen, minLen); MueLu_sumAll(comm, localMeanLen, meanLen); MueLu_maxAll(comm, localMaxLen, maxLen); @@ -1538,7 +1547,7 @@ RCP > RefMaxwell< } template -Teuchos::RCP > +Teuchos::RCP> RefMaxwell::buildProjection(const int spaceNumber, const RCP &Nullspace) const { using ATS = Kokkos::ArithTraits; using impl_Scalar = typename ATS::val_type; @@ -1779,7 +1788,7 @@ void RefMaxwell::buildNodalProlongato } template -Teuchos::RCP > +Teuchos::RCP> RefMaxwell::buildVectorNodalProlongator(const Teuchos::RCP &P_nodal) const { RCP tm = getTimer("vectorial nodal prolongator"); GetOStream(Runtime0) << solverName_ + "::compute(): building vectorial nodal prolongator" << std::endl; @@ -2004,10 +2013,10 @@ void RefMaxwell::setupSubSolve(Teucho if (params.get("Preconditioner Type") == "MueLu") { ParameterList &userParamList = params.sublist("Preconditioner Types").sublist("MueLu").sublist("user data"); if (!Nullspace.is_null()) - userParamList.set >("Nullspace", Nullspace); + userParamList.set>("Nullspace", Nullspace); if (!Material.is_null()) - userParamList.set >("Material", Material); - userParamList.set >("Coordinates", Coords); + userParamList.set>("Material", Material); + userParamList.set>("Coordinates", Coords); } thyraPrecOp = rcp(new XpetraThyraLinearOp(coarseA11_, rcp(¶ms, false))); } else @@ -2018,11 +2027,11 @@ void RefMaxwell::setupSubSolve(Teucho if (!reuse) { ParameterList &userParamList = params.sublist("user data"); if (!Coords.is_null()) - userParamList.set >("Coordinates", Coords); + userParamList.set>("Coordinates", Coords); if (!Nullspace.is_null()) - userParamList.set >("Nullspace", Nullspace); + userParamList.set>("Nullspace", Nullspace); if (!Material.is_null()) - userParamList.set >("Material", Material); + userParamList.set>("Material", Material); if (isSingular) { std::string coarseType = ""; @@ -2199,30 +2208,133 @@ void RefMaxwell::applyInverseAdditive DresTmp_->endImport(*Dres_, *Importer22_, Xpetra::INSERT); } - if (fuseProlongationAndUpdate_) { - { // prolongate (1,1) block - RCP tmP11 = getTimer("prolongation coarse (1,1) (fused)"); - P11_->apply(*P11x_, X, Teuchos::NO_TRANS, one, one); - } + { + RCP tmProlongations = getTimer("prolongations"); + + if (asyncTransfers_) { + auto tpP11 = rcp_dynamic_cast(rcp_dynamic_cast(P11_, true)->getCrsMatrix())->getTpetra_CrsMatrix(); + auto tpP11x = rcp_dynamic_cast(P11x_, true)->getTpetra_MultiVector(); + RCP> tpP11x_colmap; + auto tpX = rcp_dynamic_cast(Teuchos::rcpFromRef(X), true)->getTpetra_MultiVector(); + auto tpResidual = rcp_dynamic_cast(residual_, true)->getTpetra_MultiVector(); + + auto tpDk_1 = rcp_dynamic_cast(rcp_dynamic_cast(Dk_1_, true)->getCrsMatrix())->getTpetra_CrsMatrix(); + auto tpDx = rcp_dynamic_cast(Dx_, true)->getTpetra_MultiVector(); + RCP> tpDx_colmap; + + unsigned completedImports = 0; + std::vector completedImport(2, false); + auto tpP11importer = tpP11->getCrsGraph()->getImporter(); + if (!tpP11importer.is_null()) { + tpP11x_colmap = rcp_dynamic_cast(P11x_colmap_, true)->getTpetra_MultiVector(); + tpP11x_colmap->beginImport(*tpP11x, *tpP11importer, Tpetra::INSERT); + } - if (!onlyBoundary22_) { // prolongate (2,2) block - RCP tmD = getTimer("prolongation (2,2) (fused)"); - Dk_1_->apply(*Dx_, X, Teuchos::NO_TRANS, one, one); - } - } else { - { // prolongate (1,1) block - RCP tmP11 = getTimer("prolongation coarse (1,1) (unfused)"); - P11_->apply(*P11x_, *residual_, Teuchos::NO_TRANS); - } + RCP> tpDk_1importer; + if (!onlyBoundary22_) { + tpDk_1importer = tpDk_1->getCrsGraph()->getImporter(); + if (!tpDk_1importer.is_null()) { + tpDx_colmap = rcp_dynamic_cast(Dx_colmap_, true)->getTpetra_MultiVector(); + tpDx_colmap->beginImport(*tpDx, *tpDk_1importer, Tpetra::INSERT); + } + } else { + completedImport[1] = true; + completedImports++; + } - if (!onlyBoundary22_) { // prolongate (2,2) block - RCP tmD = getTimer("prolongation (2,2) (unfused)"); - Dk_1_->apply(*Dx_, *residual_, Teuchos::NO_TRANS, one, one); - } + while (completedImports < completedImport.size()) { + for (unsigned i = 0; i < completedImport.size(); i++) { + if (completedImport[i]) continue; + + if (i == 0) { + if (!tpP11importer.is_null()) { + if (tpP11x_colmap->transferArrived()) { + tpP11x_colmap->endImport(*tpP11x, *tpP11importer, Tpetra::INSERT); + completedImport[i] = true; + completedImports++; + + if (fuseProlongationAndUpdate_) { + RCP tmP11 = getTimer("prolongation coarse (1,1) (fused, local)"); + tpP11->localApply(*tpP11x_colmap, *tpX, Teuchos::NO_TRANS, one, one); + } else { + RCP tmP11 = getTimer("prolongation coarse (1,1) (unfused, local)"); + tpP11->localApply(*tpP11x_colmap, *tpResidual, Teuchos::NO_TRANS); + } + } + } else { + completedImport[i] = true; + completedImports++; + + if (fuseProlongationAndUpdate_) { + RCP tmP11 = getTimer("prolongation coarse (1,1) (fused, local)"); + tpP11->localApply(*tpP11x, *tpX, Teuchos::NO_TRANS, one, one); + } else { + RCP tmP11 = getTimer("prolongation coarse (1,1) (unfused, local)"); + tpP11->localApply(*tpP11x, *tpResidual, Teuchos::NO_TRANS); + } + } + } else { + if (!tpDk_1importer.is_null()) { + if (tpDx_colmap->transferArrived()) { + tpDx_colmap->endImport(*tpDx, *tpDk_1importer, Tpetra::INSERT); + completedImport[i] = true; + completedImports++; + + if (fuseProlongationAndUpdate_) { + RCP tmD = getTimer("prolongation (2,2) (fused, local)"); + tpDk_1->localApply(*tpDx_colmap, *tpX, Teuchos::NO_TRANS, one, one); + } else { + RCP tmD = getTimer("prolongation (2,2) (unfused, local)"); + tpDk_1->localApply(*tpDx_colmap, *tpResidual, Teuchos::NO_TRANS, one, one); + } + } + } else { + completedImport[i] = true; + completedImports++; + + if (fuseProlongationAndUpdate_) { + RCP tmD = getTimer("prolongation (2,2) (fused, local)"); + tpDk_1->localApply(*tpDx, *tpX, Teuchos::NO_TRANS, one, one); + } else { + RCP tmD = getTimer("prolongation (2,2) (unfused, local)"); + tpDk_1->localApply(*tpDx, *tpResidual, Teuchos::NO_TRANS, one, one); + } + } + } + } + } + + if (!fuseProlongationAndUpdate_) { // update current solution + RCP tmUpdate = getTimer("update"); + X.update(one, *residual_, one); + } + } else { + if (fuseProlongationAndUpdate_) { + { // prolongate (1,1) block + RCP tmP11 = getTimer("prolongation coarse (1,1) (fused)"); + P11_->apply(*P11x_, X, Teuchos::NO_TRANS, one, one); + } + + if (!onlyBoundary22_) { // prolongate (2,2) block + RCP tmD = getTimer("prolongation (2,2) (fused)"); + Dk_1_->apply(*Dx_, X, Teuchos::NO_TRANS, one, one); + } + } else { + { // prolongate (1,1) block + RCP tmP11 = getTimer("prolongation coarse (1,1) (unfused)"); + P11_->apply(*P11x_, *residual_, Teuchos::NO_TRANS); + } - { // update current solution - RCP tmUpdate = getTimer("update"); - X.update(one, *residual_, one); + if (!onlyBoundary22_) { // prolongate (2,2) block + RCP tmD = getTimer("prolongation (2,2) (unfused)"); + Dk_1_->apply(*Dx_, *residual_, Teuchos::NO_TRANS, one, one); + } + + { // update current solution + RCP tmUpdate = getTimer("update"); + X.update(one, *residual_, one); + } + } } } } @@ -2374,44 +2486,44 @@ RefMaxwell:: RCP NodalCoords; Dk_1 = pop(List, "Dk_1", Dk_1); - Dk_2 = pop >(List, "Dk_2", Dk_2); - D0 = pop >(List, "D0", D0); + Dk_2 = pop>(List, "Dk_2", Dk_2); + D0 = pop>(List, "D0", D0); - M1_beta = pop >(List, "M1_beta", M1_beta); - M1_alpha = pop >(List, "M1_alpha", M1_alpha); + M1_beta = pop>(List, "M1_beta", M1_beta); + M1_alpha = pop>(List, "M1_alpha", M1_alpha); - Mk_one = pop >(List, "Mk_one", Mk_one); - Mk_1_one = pop >(List, "Mk_1_one", Mk_1_one); + Mk_one = pop>(List, "Mk_one", Mk_one); + Mk_1_one = pop>(List, "Mk_1_one", Mk_1_one); - invMk_1_invBeta = pop >(List, "invMk_1_invBeta", invMk_1_invBeta); - invMk_2_invAlpha = pop >(List, "invMk_2_invAlpha", invMk_2_invAlpha); + invMk_1_invBeta = pop>(List, "invMk_1_invBeta", invMk_1_invBeta); + invMk_2_invAlpha = pop>(List, "invMk_2_invAlpha", invMk_2_invAlpha); - Nullspace11 = pop >(List, "Nullspace11", Nullspace11); - Nullspace22 = pop >(List, "Nullspace22", Nullspace22); - NodalCoords = pop >(List, "Coordinates", NodalCoords); + Nullspace11 = pop>(List, "Nullspace11", Nullspace11); + Nullspace22 = pop>(List, "Nullspace22", Nullspace22); + NodalCoords = pop>(List, "Coordinates", NodalCoords); // old parameter names - if (List.isType >("Ms")) { + if (List.isType>("Ms")) { if (M1_beta.is_null()) - M1_beta = pop >(List, "Ms"); + M1_beta = pop>(List, "Ms"); else TEUCHOS_ASSERT(false); } - if (List.isType >("M1")) { + if (List.isType>("M1")) { if (Mk_one.is_null()) - Mk_one = pop >(List, "M1"); + Mk_one = pop>(List, "M1"); else TEUCHOS_ASSERT(false); } - if (List.isType >("M0inv")) { + if (List.isType>("M0inv")) { if (invMk_1_invBeta.is_null()) - invMk_1_invBeta = pop >(List, "M0inv"); + invMk_1_invBeta = pop>(List, "M0inv"); else TEUCHOS_ASSERT(false); } - if (List.isType >("Nullspace")) { + if (List.isType>("Nullspace")) { if (Nullspace11.is_null()) - Nullspace11 = pop >(List, "Nullspace"); + Nullspace11 = pop>(List, "Nullspace"); else TEUCHOS_ASSERT(false); } @@ -2680,7 +2792,7 @@ void RefMaxwell:: describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel /* verbLevel */) const { std::ostringstream oss; - RCP > comm = SM_Matrix_->getDomainMap()->getComm(); + RCP> comm = SM_Matrix_->getDomainMap()->getComm(); #ifdef HAVE_MPI int root; @@ -2749,8 +2861,8 @@ void RefMaxwell:: std::string outstr = oss.str(); #ifdef HAVE_MPI - RCP > mpiComm = rcp_dynamic_cast >(comm); - MPI_Comm rawComm = (*mpiComm->getRawMpiComm())(); + RCP> mpiComm = rcp_dynamic_cast>(comm); + MPI_Comm rawComm = (*mpiComm->getRawMpiComm())(); int strLength = outstr.size(); MPI_Bcast(&strLength, 1, MPI_INT, root, rawComm); @@ -2776,9 +2888,9 @@ void RefMaxwell:: int numProcs = comm->getSize(); #ifdef HAVE_MPI - RCP > tmpic = rcp_dynamic_cast >(comm); + RCP> tmpic = rcp_dynamic_cast>(comm); TEUCHOS_TEST_FOR_EXCEPTION(tmpic == Teuchos::null, Exceptions::RuntimeError, "Cannot cast base Teuchos::Comm to Teuchos::MpiComm object."); - RCP > rawMpiComm = tmpic->getRawMpiComm(); + RCP> rawMpiComm = tmpic->getRawMpiComm(); #endif char status = 0;