diff --git a/src/sst/elements/iris/sumi/sim_transport.cc b/src/sst/elements/iris/sumi/sim_transport.cc index b22fb780dd..3af60045a7 100644 --- a/src/sst/elements/iris/sumi/sim_transport.cc +++ b/src/sst/elements/iris/sumi/sim_transport.cc @@ -172,7 +172,6 @@ SimTransport::SimTransport(SST::Params& params, SST::Hg::App* parent) : Library(params, parent), //the server is what takes on the specified libname completion_queues_(1), -// spy_bytes_(nullptr), default_progress_queue_(parent->os()), nic_ioctl_(parent->os()->nicDataIoctl()), qos_analysis_(nullptr), @@ -206,13 +205,6 @@ SimTransport::SimTransport(SST::Params& params, SST::Hg::App* parent) : rdma_page_delay_ = TimeDelta(params.find("rdma_page_delay", "0s").getValue().toDouble()); pin_delay_ = rdma_pin_latency_.ticks() || rdma_page_delay_.ticks(); page_size_ = params.find("rdma_page_size", "4096").getRoundedValue(); - - // std::cerr << "post_rdma_delay: " << post_rdma_delay_ << std::endl; - // std::cerr << "post_header_delay: " << post_header_delay_ << std::endl; - // std::cerr << "poll_delay: " << poll_delay_ << std::endl; - // std::cerr << "rdma_pin_latency " << rdma_pin_latency_ << std::endl; - // std::cerr << "rdma_page_delay " << rdma_page_delay_ << std::endl; - // std::cerr << "rdma_page_size " << page_size_ << std::endl; output.output("%d", sid().app_); nproc_ = os_->nranks(); @@ -277,9 +269,6 @@ SimTransport::~SimTransport() if (del) delete server; if (engine_) delete engine_; - - //if (spy_bytes_) delete spy_bytes_; - //if (spy_num_messages_) delete spy_num_messages_; } void @@ -319,6 +308,7 @@ SimTransport::nidlist() const //the types are the same size and the bits can be //interpreted correctly //return (int*) rank_mapper_->rankToNode().data(); + sst_hg_abort_printf("sumi_transport::nidList unimplemented"); return nullptr; } diff --git a/src/sst/elements/mercury/Makefile.am b/src/sst/elements/mercury/Makefile.am index 36289252cd..5b1db31ee2 100644 --- a/src/sst/elements/mercury/Makefile.am +++ b/src/sst/elements/mercury/Makefile.am @@ -23,6 +23,8 @@ libhg_la_SOURCES = \ common/connection.cc \ components/nic.cc \ components/node.cc \ + components/node_base.cc \ + components/node_CL.cc \ components/operating_system.cc \ hardware/common/packet.cc \ hardware/common/recv_cq.cc \ @@ -118,6 +120,9 @@ nobase_library_include_HEADERS = \ components/nic.h \ components/nic_fwd.h \ components/node.h \ + components/node_base.h \ + components/node_base_fwd.h \ + components/node_CL.h \ components/node_fwd.h \ components/operating_system.h \ components/operating_system_fwd.h \ diff --git a/src/sst/elements/mercury/components/nic.cc b/src/sst/elements/mercury/components/nic.cc index a21b8468dd..9ab8eb9794 100644 --- a/src/sst/elements/mercury/components/nic.cc +++ b/src/sst/elements/mercury/components/nic.cc @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include @@ -34,7 +34,7 @@ NicEvent::serialize_order(serializer &ser) ser & msg_; } -NIC::NIC(uint32_t id, SST::Params& params, Node* parent) : +NIC::NIC(uint32_t id, SST::Params& params, NodeBase* parent) : SST::Hg::SubComponent(id), parent_(parent), my_addr_(parent->os()->addr()), @@ -404,7 +404,7 @@ NIC::internodeSend(NetworkMessage* netmsg) void NIC::sendToNode(NetworkMessage* payload) { - auto forward_ev = newCallback(parent_, &Node::handle, payload); + auto forward_ev = newCallback(parent_, &NodeBase::handle, payload); parent_->sendExecutionEventNow(forward_ev); } diff --git a/src/sst/elements/mercury/components/nic.h b/src/sst/elements/mercury/components/nic.h index 3383b18593..1ef1762825 100644 --- a/src/sst/elements/mercury/components/nic.h +++ b/src/sst/elements/mercury/components/nic.h @@ -21,12 +21,12 @@ #include #include +#include +#include #include #include #include #include -#include -#include #include #include #include @@ -69,7 +69,7 @@ class NIC : public SST::Hg::SubComponent public: SST_ELI_REGISTER_SUBCOMPONENT_API(SST::Hg::NIC, - SST::Hg::Node*) + SST::Hg::NodeBase*) SST_ELI_REGISTER_SUBCOMPONENT( NIC, @@ -210,11 +210,11 @@ class NIC : public SST::Hg::SubComponent } public: - NIC(uint32_t id, SST::Params& params, SST::Hg::Node* parent); + NIC(uint32_t id, SST::Params& params, SST::Hg::NodeBase* parent); protected: - Node* parent() const { + SST::Hg::NodeBase* parent() const { return parent_; } @@ -224,7 +224,7 @@ class NIC : public SST::Hg::SubComponent protected: int negligibleSize_; - Node* parent_; + SST::Hg::NodeBase* parent_; NodeId my_addr_; EventLink::ptr logp_link_; diff --git a/src/sst/elements/mercury/components/node.cc b/src/sst/elements/mercury/components/node.cc index 1b21526104..ecc98d7b38 100644 --- a/src/sst/elements/mercury/components/node.cc +++ b/src/sst/elements/mercury/components/node.cc @@ -25,60 +25,7 @@ extern template class HgBase; extern template SST::TimeConverter* HgBase::time_converter_; Node::Node(ComponentId_t id, Params ¶ms) - : SST::Hg::Component(id), nic_(0) { - my_addr_ = params.find("logicalID",-1); - unsigned int verbose = params.find("verbose",0); - out_ = std::unique_ptr(new SST::Output(sprintf("Node%d:",my_addr_), verbose, 0, Output::STDOUT)); - - out_->debug(CALL_INFO, 1, 0, "loading hg.operatingsystem\n"); - os_ = loadUserSubComponent("os_slot", SST::ComponentInfo::SHARE_NONE, this); - assert(os_); - - out_->debug(CALL_INFO, 1, 0, "loading hg.NIC\n"); - link_control_ = loadUserSubComponent("link_control_slot", SST::ComponentInfo::SHARE_NONE,1); - if (link_control_) { - out_->debug(CALL_INFO, 1, 0, "loading hg.NIC\n"); - nic_ = loadUserSubComponent("nic_slot", SST::ComponentInfo::SHARE_NONE, this); - assert(nic_); - nic_->set_link_control(link_control_); - } - else { - // assume basic tests - // (unused but needs to be there or multithread termination breaks) - netLink_ = configureLink("network"); - } - - unsigned int nranks = params.find("nranks",-1); - os_->set_nranks(nranks); - - int ncores_ = params.find("ncores", 1); - int nsockets_ = params.find("nsockets",1); - - // Tell the simulation not to end until we're ready - registerAsPrimaryComponent(); - primaryComponentDoNotEndSim(); -} - -void -Node::init(unsigned int phase) -{ - SST::Component::init(phase); - os_->init(phase); - if (nic_) nic_->init(phase); -} - -void -Node::setup() -{ - SST::Component::setup(); - os_->setup(); - if (nic_) nic_->setup(); -} - -void -Node::handle(Request* req) -{ - os_->handleRequest(req); + : SST::Hg::NodeBase(id,params) { } } // namespace Hg diff --git a/src/sst/elements/mercury/components/node.h b/src/sst/elements/mercury/components/node.h index 83af9217bb..70662b835e 100644 --- a/src/sst/elements/mercury/components/node.h +++ b/src/sst/elements/mercury/components/node.h @@ -17,6 +17,7 @@ #include +#include #include #include #include @@ -30,7 +31,7 @@ namespace SST { namespace Hg { // Components inherit from SST::Component -class Node : public SST::Hg::Component { +class Node : public NodeBase { public: /* * SST Registration macros register Components with the SST Core and @@ -40,69 +41,17 @@ class Node : public SST::Hg::Component { */ // REGISTER THIS COMPONENT INTO THE ELEMENT LIBRARY SST_ELI_REGISTER_COMPONENT( - Node, // Component class + SST::Hg::Node, // Component class "hg", // Component library (for Python/library lookup) - "node", // Component name (for Python/library lookup) + "Node", // Component name (for Python/library lookup) SST_ELI_ELEMENT_VERSION( 0, 0, 1), // Version of the component (not related to SST version) - "Mercury Node", // Description - COMPONENT_CATEGORY_UNCATEGORIZED // Category + "Simple Mercury node", // Description + COMPONENT_CATEGORY_UNCATEGORIZED, // Category + SST::Hg::NodeBase ) - SST_ELI_DOCUMENT_PARAMS({"verbose", - "Output verbose level", 0}, - ) - - SST_ELI_DOCUMENT_PORTS( - {"network", "Dummy network port to connect nodes for testing", {} }, - ) - - SST_ELI_DOCUMENT_SUBCOMPONENT_SLOTS( - {"os_slot", "The operating system", "hg.operating_system"}, - {"nic_slot", "The nic", "hg.nic"}, - {"link_control_slot", "Slot for a link control", "SST::Interfaces::SimpleNetwork" } - ) - Node(SST::ComponentId_t id, SST::Params ¶ms); - - /** - @return A unique integer identifier - */ - NodeId addr() const { - return my_addr_; - } - - void init(unsigned int phase) override; - - void setup() override; - - void endSim() { - primaryComponentOKToEndSim(); - } - - SST::Hg::OperatingSystem* os() const { - return os_; - } - - int ncores() { return ncores_; } - int nsockets() { return nsockets_; } - - void handle(Request* req); - - SST::Hg::NIC* nic() { return nic_; } - - std::string toString() { return sprintf("HgNode%d:",my_addr_); } - -private: - - SST::Hg::NIC* nic_; - SST::Hg::OperatingSystem* os_; - SST::Interfaces::SimpleNetwork* link_control_; - SST::Link* netLink_; - std::unique_ptr out_; - NodeId my_addr_; - int ncores_; - int nsockets_; }; } // namespace Hg diff --git a/src/sst/elements/mercury/components/node_CL.cc b/src/sst/elements/mercury/components/node_CL.cc new file mode 100644 index 0000000000..c45fd0e862 --- /dev/null +++ b/src/sst/elements/mercury/components/node_CL.cc @@ -0,0 +1,31 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#include + +namespace SST { +namespace Hg { + +extern template class HgBase; +extern template SST::TimeConverter* HgBase::time_converter_; + +NodeCL::NodeCL(ComponentId_t id, Params ¶ms) + : NodeBase(id,params) { + int ncores_ = params.find("ncores", 1); + int nsockets_ = params.find("nsockets",1); +} + +} // namespace Hg +} // namespace SST diff --git a/src/sst/elements/mercury/components/node_CL.h b/src/sst/elements/mercury/components/node_CL.h new file mode 100644 index 0000000000..ecd90b1f0c --- /dev/null +++ b/src/sst/elements/mercury/components/node_CL.h @@ -0,0 +1,64 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace SST { +namespace Hg { + +// Components inherit from SST::Component +class NodeCL : public NodeBase { +public: + /* + * SST Registration macros register Components with the SST Core and + * document their parameters, ports, etc. + * SST_ELI_REGISTER_COMPONENT is required, the documentation macros + * are only required if relevant + */ + // REGISTER THIS COMPONENT INTO THE ELEMENT LIBRARY + SST_ELI_REGISTER_COMPONENT( + SST::Hg::NodeCL, // Component class + "hg", // Component library (for Python/library lookup) + "NodeCL", // Component name (for Python/library lookup) + SST_ELI_ELEMENT_VERSION( + 0, 0, 1), // Version of the component (not related to SST version) + "Mercury Node including ComputeLibrary", // Description + COMPONENT_CATEGORY_UNCATEGORIZED, // Category + SST::Hg::Node + ) + + NodeCL(SST::ComponentId_t id, SST::Params ¶ms); + + int ncores() { return ncores_; } + int nsockets() { return nsockets_; } + + std::string toString() override { return sprintf("HgNode%d:",my_addr_); } + +private: + + int ncores_; + int nsockets_; +}; + +} // namespace Hg +} // namespace SST diff --git a/src/sst/elements/mercury/components/node_base.cc b/src/sst/elements/mercury/components/node_base.cc new file mode 100644 index 0000000000..9f4a32fae5 --- /dev/null +++ b/src/sst/elements/mercury/components/node_base.cc @@ -0,0 +1,82 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#include +#include +#include +#include + +namespace SST { +namespace Hg { + +extern template class HgBase; +extern template SST::TimeConverter* HgBase::time_converter_; + +NodeBase::NodeBase(ComponentId_t id, Params ¶ms) + : SST::Hg::Component(id), nic_(0) { + my_addr_ = params.find("logicalID",-1); + unsigned int verbose = params.find("verbose",0); + out_ = std::unique_ptr(new SST::Output(sprintf("Node%d:",my_addr_), verbose, 0, Output::STDOUT)); + + out_->debug(CALL_INFO, 1, 0, "loading hg.operatingsystem\n"); + os_ = loadUserSubComponent("os_slot", SST::ComponentInfo::SHARE_NONE, this); + assert(os_); + + out_->debug(CALL_INFO, 1, 0, "loading hg.NIC\n"); + link_control_ = loadUserSubComponent("link_control_slot", SST::ComponentInfo::SHARE_NONE,1); + if (link_control_) { + out_->debug(CALL_INFO, 1, 0, "loading hg.NIC\n"); + nic_ = loadUserSubComponent("nic_slot", SST::ComponentInfo::SHARE_NONE, this); + assert(nic_); + nic_->set_link_control(link_control_); + } + else { + // assume basic tests + // (unused but needs to be there or multithread termination breaks) + netLink_ = configureLink("network"); + } + + unsigned int nranks = params.find("nranks",-1); + os_->set_nranks(nranks); + + // Tell the simulation not to end until we're ready + registerAsPrimaryComponent(); + primaryComponentDoNotEndSim(); +} + +void +NodeBase::init(unsigned int phase) +{ + SST::Component::init(phase); + os_->init(phase); + if (nic_) nic_->init(phase); +} + +void +NodeBase::setup() +{ + SST::Component::setup(); + os_->setup(); + if (nic_) nic_->setup(); +} + +void +NodeBase::handle(Request* req) +{ + os_->handleRequest(req); +} + +} // namespace Hg +} // namespace SST diff --git a/src/sst/elements/mercury/components/node_base.h b/src/sst/elements/mercury/components/node_base.h new file mode 100644 index 0000000000..c340e8c54d --- /dev/null +++ b/src/sst/elements/mercury/components/node_base.h @@ -0,0 +1,90 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SST { +namespace Hg { + +class NodeBase : public SST::Hg::Component { +public: + + // Base class for node components + SST_ELI_REGISTER_COMPONENT_BASE(SST::Hg::NodeBase) + + SST_ELI_DOCUMENT_PARAMS({"verbose", + "Output verbose level", 0}, + ) + + SST_ELI_DOCUMENT_PORTS( + {"network", "Dummy network port to connect nodes for testing", {} }, + ) + + SST_ELI_DOCUMENT_SUBCOMPONENT_SLOTS( + {"os_slot", "The operating system", "hg.operating_system"}, + {"nic_slot", "The nic", "hg.nic"}, + {"link_control_slot", "Slot for a link control", "SST::Interfaces::SimpleNetwork" } + ) + + NodeBase(SST::ComponentId_t id, SST::Params ¶ms); + + /** + @return A unique integer identifier + */ + NodeId addr() const { + return my_addr_; + } + + void init(unsigned int phase) override; + + void setup() override; + + void endSim() { + primaryComponentOKToEndSim(); + } + + SST::Hg::OperatingSystem* os() const { + return os_; + } + + void handle(Request* req); + + SST::Hg::NIC* nic() { return nic_; } + + virtual std::string toString() { return sprintf("HgNode%d:",my_addr_); } + +protected: + + SST::Hg::NIC* nic_; + SST::Hg::OperatingSystem* os_; + SST::Interfaces::SimpleNetwork* link_control_; + SST::Link* netLink_; + std::unique_ptr out_; + NodeId my_addr_; +}; + +} // namespace Hg +} // namespace SST diff --git a/src/sst/elements/mercury/components/node_base_fwd.h b/src/sst/elements/mercury/components/node_base_fwd.h new file mode 100644 index 0000000000..4c091a34a6 --- /dev/null +++ b/src/sst/elements/mercury/components/node_base_fwd.h @@ -0,0 +1,24 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +namespace SST { +namespace Hg { + +class NodeBase; + +} +} diff --git a/src/sst/elements/mercury/components/operating_system.cc b/src/sst/elements/mercury/components/operating_system.cc index 4cd5b9942a..e4dc354c04 100644 --- a/src/sst/elements/mercury/components/operating_system.cc +++ b/src/sst/elements/mercury/components/operating_system.cc @@ -65,7 +65,7 @@ class DeleteThreadEvent : Thread* thr_; }; -OperatingSystem::OperatingSystem(SST::ComponentId_t id, SST::Params& params, Node* parent) : +OperatingSystem::OperatingSystem(SST::ComponentId_t id, SST::Params& params, NodeBase* parent) : SST::Hg::SubComponent(id), node_(parent), des_context_(nullptr), @@ -116,10 +116,10 @@ OperatingSystem::OperatingSystem(SST::ComponentId_t id, SST::Params& params, Nod app_launcher_ = new AppLauncher(this); addLaunchRequests(params); - compute_sched_ = SST::Hg::create( - "hg", params.find("compute_scheduler", "simple"), - params, this, node_ ? node_->ncores() : 1, node_ ? node_->nsockets() - : 1); + // compute_sched_ = SST::Hg::create( + // "hg", params.find("compute_scheduler", "simple"), + // params, this, node_ ? node_->ncores() : 1, node_ ? node_->nsockets() + // : 1); StackAlloc::init(params); initThreading(params); diff --git a/src/sst/elements/mercury/components/operating_system.h b/src/sst/elements/mercury/components/operating_system.h index f08ad33e50..4a133dece6 100644 --- a/src/sst/elements/mercury/components/operating_system.h +++ b/src/sst/elements/mercury/components/operating_system.h @@ -20,7 +20,7 @@ #include #include -#include +#include #include #include #include @@ -47,7 +47,7 @@ class OperatingSystem : public SST::Hg::SubComponent { public: SST_ELI_REGISTER_SUBCOMPONENT_API(SST::Hg::OperatingSystem, - SST::Hg::Node*) + SST::Hg::NodeBase*) SST_ELI_REGISTER_SUBCOMPONENT( OperatingSystem, @@ -58,7 +58,7 @@ class OperatingSystem : public SST::Hg::SubComponent { SST::Hg::OperatingSystem ) - OperatingSystem(SST::ComponentId_t id, SST::Params& params, Node* parent); + OperatingSystem(SST::ComponentId_t id, SST::Params& params, NodeBase* parent); virtual ~OperatingSystem(); @@ -153,7 +153,7 @@ class OperatingSystem : public SST::Hg::SubComponent { unsigned int verbose_; int nranks_; - Node* node_; + SST::Hg::NodeBase* node_; Thread* active_thread_; Thread* blocked_thread_; int next_condition_; @@ -193,7 +193,7 @@ class OperatingSystem : public SST::Hg::SubComponent { public: - Node* node() const { + NodeBase* node() const { return node_; } diff --git a/src/sst/elements/mercury/libraries/compute/compute_api.h b/src/sst/elements/mercury/libraries/compute/compute_api.h index de5504bf9a..cfbd821f19 100644 --- a/src/sst/elements/mercury/libraries/compute/compute_api.h +++ b/src/sst/elements/mercury/libraries/compute/compute_api.h @@ -24,6 +24,8 @@ class ComputeAPI virtual void sleep(TimeDelta time) = 0; virtual void compute(TimeDelta time) = 0; virtual void computeBlockMemcpy(uint64_t bytes) = 0; + virtual void write(uint64_t bytes) = 0; + virtual void copy(uint64_t bytes) = 0; }; } // end namespace Hg diff --git a/src/sst/elements/mercury/libraries/compute/compute_library.cc b/src/sst/elements/mercury/libraries/compute/compute_library.cc index 04a826fdc9..6d99462f5e 100644 --- a/src/sst/elements/mercury/libraries/compute/compute_library.cc +++ b/src/sst/elements/mercury/libraries/compute/compute_library.cc @@ -13,6 +13,7 @@ // information, see the LICENSE file in the top level directory of the // distribution. +#include #include #include #include @@ -20,20 +21,100 @@ namespace SST { namespace Hg { -void -ComputeLibrary::compute(TimeDelta time) -{ +ComputeLibrary::ComputeLibrary(SST::Params ¶ms, App *parent) + : Library(params, parent), parent_os_(parent->os()) { + + access_width_bytes_ = params.find("lib_compute_access_width", 64) / 8; + + if (params.contains("compute_library_unroll_loops")) { + double loop_unroll = params.find("compute_library_unroll_loops"); + loop_overhead_ = 1.0 / loop_unroll; + } else { + loop_overhead_ = params.find("compute_library_loop_overhead", 1.0); + } +} + +void +ComputeLibrary::compute(TimeDelta time) { if (time.sec() < 0) { sst_hg_abort_printf("ComputeLibrary can't compute for less than zero time"); } - parent()->os()->blockTimeout(time); + parent_os_->blockTimeout(time); } void ComputeLibrary::sleep(TimeDelta time) { - parent()->os()->blockTimeout(time); + parent_os_->blockTimeout(time); +} + +void ComputeLibrary::computeBlockMemcpy(uint64_t bytes) { + copy(bytes); +} + +void +ComputeLibrary::write(uint64_t bytes) +{ + doAccess(bytes); +} + +void +ComputeLibrary::copy(uint64_t bytes) +{ + doAccess(bytes); +} + +void +ComputeLibrary::doAccess(uint64_t bytes) +{ + uint64_t num_loops = bytes / access_width_bytes_; + int nflops = 0; + int nintops = 1; //memmove instruction + computeLoop(num_loops, nflops, nintops, access_width_bytes_); +} + +void +ComputeLibrary::computeLoop(uint64_t num_loops, + uint32_t flops_per_loop, + uint32_t nintops_per_loop, + uint32_t bytes_per_loop) +{ + /** Configure the compute request */ + uint64_t loop_control_ops = 2 * num_loops * loop_overhead_; + uint64_t num_intops = loop_control_ops + nintops_per_loop*num_loops; + computeDetailed( + flops_per_loop*num_loops, + num_intops, + bytes_per_loop*num_loops, 1); +} + +void +ComputeLibrary::computeDetailed( + uint64_t flops, + uint64_t nintops, + uint64_t bytes, + int nthread) +{ + // /** Configure the compute request */ + // auto cmsg = new ComputeEvent_impl; + // basic_instructions_st& st = cmsg->data(); + // st.flops = flops; + // st.intops = nintops; + // st.mem_sequential = bytes; + // st.nthread = nthread; + + // // Do not overwrite an existing tag + // FTQScope scope(os_->activeThread(), FTQTag::compute); + + //computeInst(cmsg, nthread); + // delete cmsg; } +// void +// LibCompute::computeInst(ComputeEvent* cmsg, int nthr) +// { +// //os_->execute(ami::COMP_INSTR, cmsg, nthr); +// } + } // end namespace Hg } // end namespace SST diff --git a/src/sst/elements/mercury/libraries/compute/compute_library.h b/src/sst/elements/mercury/libraries/compute/compute_library.h index 56cb7d061e..a444195f94 100644 --- a/src/sst/elements/mercury/libraries/compute/compute_library.h +++ b/src/sst/elements/mercury/libraries/compute/compute_library.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -36,14 +37,32 @@ class ComputeLibrary : public ComputeAPI, public Library ~ComputeLibrary() override{} - ComputeLibrary(SST::Params& params, - App* parent) - : Library(params,parent) { - } + ComputeLibrary(SST::Params& params, App* parent); void sleep(TimeDelta time) override; + void compute(TimeDelta time) override; - void computeBlockMemcpy(uint64_t bytes) override { } + + void computeBlockMemcpy(uint64_t bytes) override; + + void write(uint64_t bytes) override; + + void copy(uint64_t bytes) override; + + private: + + void doAccess(uint64_t bytes); + + void computeLoop(uint64_t num_loops, uint32_t flops_per_loop, + uint32_t nintops_per_loop, uint32_t bytes_per_loop); + + void computeDetailed(uint64_t flops, uint64_t nintops, uint64_t bytes, + int nthread); + +private: + OperatingSystem* parent_os_; + int access_width_bytes_; + double loop_overhead_; }; } // end namespace Hg diff --git a/src/sst/elements/mercury/libraries/compute/compute_scheduler.cc b/src/sst/elements/mercury/libraries/compute/compute_scheduler.cc new file mode 100644 index 0000000000..023f9fa971 --- /dev/null +++ b/src/sst/elements/mercury/libraries/compute/compute_scheduler.cc @@ -0,0 +1,86 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +#include + +namespace SST { +namespace Hg { + +ComputeScheduler(SST::Params ¶ms, SST::Hg::OperatingSystem *os) + : os_(os) { + ncores_ = params.find("lib_compute_ncores", 1); + nsocket_ = params.find("lib_compute_nsocket", 1); +} + +/** + * @brief reserve_core + * @param thr The physical thread requesting to compute + */ +void ComputeScheduler::reserveCores(int ncores, Thread *thr) { + int total_cores_needed = ncores + ncore_active_; + while (total_cores_needed > ncores_) { + // debug_printf(sprockit::dbg::compute_scheduler, + // "Need %d cores, have %d for thread %ld - blocking", + // ncores, ncores_ - ncore_active_, thr->threadId()); + pending_threads_.emplace_back(ncores, thr); + os_->block(); + // we can accidentally unblock due to "race" conditions + // reset the core check to make sure we have what we need + total_cores_needed = ncores + ncore_active_; + } + // debug_printf(sprockit::dbg::compute_scheduler, + // "Reserved %d cores for thread %ld", + // ncores, thr->threadId()); + // no worrying about masks + for (int i = ncore_active_; i < ncore_active_ + ncores; ++i) { + thr->addActiveCore(i); + } + ncore_active_ += ncores; +} + +void ComputeScheduler::releaseCores(int ncore, Thread *thr) { + ncore_active_ -= ncores; + // debug_printf(sprockit::dbg::compute_scheduler, + // "Released %d cores for thread %ld - now %d active, %d free", + // ncores, thr->threadId(), ncore_active_, ncores_ - ncore_active_); + for (int i = 0; i < ncores; ++i) { + thr->popActiveCore(); + } + + int nfree_cores = ncores_ - ncore_active_; + Thread *to_unblock = nullptr; + for (auto iter = pending_threads_.begin(); iter != pending_threads_.end(); + ++iter) { + auto pair = *iter; + int ncores_needed = pair.first; + // debug_printf(sprockit::dbg::compute_scheduler, + // "Thread %d trying to restart thread %d with %d free cores - need %d", + // thr->threadId(), pair.second->threadId(), + // nfree_cores, ncores_needed); + if (nfree_cores >= ncores_needed) { + pending_threads_.erase(iter); + to_unblock = pair.second; + break; + } + } + if (to_unblock) { + os_->unblock(to_unblock); + } +} + +} // end namespace Hg +} // end namespace SST diff --git a/src/sst/elements/mercury/libraries/compute/compute_scheduler.h b/src/sst/elements/mercury/libraries/compute/compute_scheduler.h new file mode 100644 index 0000000000..337b133f43 --- /dev/null +++ b/src/sst/elements/mercury/libraries/compute/compute_scheduler.h @@ -0,0 +1,61 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +#include +#include +#include + +namespace SST { +namespace Hg { + +class ComputeScheduler : +{ + ComputeScheduler(SST::Params& params, SST::Hg::OperatingSystem* os); + + virtual ~ComputeScheduler() {} + + + int ncores() const { + return ncores_; + } + + int nsocket() const { + return nsocket_; + } + + /** + * @brief reserve_core + * @param thr The physical thread requesting to compute + */ + void reserveCores(int ncore, Thread* thr) override; + + void releaseCores(int ncore, Thread* thr) override; + + + protected: + int ncores_; + int nsocket_; + int cores_per_socket_; + int ncore_active_; + SST::Hg::OperatingSystem* os_; + std::list> pending_threads_; + + +}; + +} // end namespace Hg +} // end namespace SST diff --git a/src/sst/elements/mercury/libraries/compute/compute_scheduler_api.h b/src/sst/elements/mercury/libraries/compute/compute_scheduler_api.h new file mode 100644 index 0000000000..b88d54896a --- /dev/null +++ b/src/sst/elements/mercury/libraries/compute/compute_scheduler_api.h @@ -0,0 +1,30 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +namespace SST { +namespace Hg { + +class ComputeSchedulerAPI +{ + int ncores() const = 0; + int nsocket() const = 0; + virtual void reserveCores(int ncore, Thread* thr) = 0; + virtual void releaseCores(int ncore, Thread* thr) = 0; +}; + +} // end namespace Hg +} // end namespace SST diff --git a/src/sst/elements/mercury/libraries/compute/memory_model.cc b/src/sst/elements/mercury/libraries/compute/memory_model.cc new file mode 100644 index 0000000000..42062e5316 --- /dev/null +++ b/src/sst/elements/mercury/libraries/compute/memory_model.cc @@ -0,0 +1,115 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#include +#include + +namespace SST { +namespace Hg { +namespace ComputeLibrary { + +SnapprMemoryModel::SnapprMemoryModel(uint32_t id, SST::Params ¶ms, Node* parent) : + MemoryModel(id, params, parent), + flowId_(0), + channelInterleaver_(0) +{ + flow_mtu_ = params.find("flow_mtu", "512").getRoundedValue(); + + mtu_ = params.find("flow_mtu", "4096").getRoundedValue(); + + auto max_bw = params.find("channel_bandwidth"); + channel_byte_delay_ = TimeDelta(max_bw.getValue().inverse().toDouble()); + + int num_channels = params.find("num_channels"); + channels_.resize(num_channels); + for (ChannelQueue& q : channels_){ + q.byte_delay = channel_byte_delay_; + } + + flow_rsp_id_ = initialize( makeHandler(this, &SnapprMemoryModel::flowRequestResponse) ); +} + + +void +SnapprMemoryModel::accessFlow(uint64_t bytes, TimeDelta byte_request_delay, Callback *cb) +{ + if (bytes == 0){ + sendExecutionEventNow(cb); + return; + } + + uint32_t flowId = flowId_++; + debug("Starting flow of size %" PRIu64 " on ID %" PRIu32 " with request delay %10.5e", + bytes, flowId, byte_request_delay.sec()); + uint32_t initial_bytes = bytes % flow_mtu_; + if (initial_bytes == 0){ + initial_bytes = flow_mtu_; + } + + FlowRequest* req = new FlowRequest; + req->bytes = initial_bytes; + req->flowId = flowId; + auto* ev = newCallback(this, &SnapprMemoryModel::accessRequest, flow_rsp_id_, req); + TimeDelta delay = byte_request_delay * initial_bytes; + sendDelayedExecutionEvent(delay, ev); + + Flow& f = flows_[flowId]; + f.bytesLeft = bytes - initial_bytes; + f.callback = cb; + f.byteRequestDelay = byte_request_delay; +} + +void +SnapprMemoryModel::flowRequestResponse(Request* req) +{ + FlowRequest* freq = static_cast(req); + Flow& f = flows_[freq->flowId]; + if (f.bytesLeft == 0){ + debug("Receive %" PRIu32 " bytes of flow %" PRIu64 ": completing flow", + req->bytes, freq->flowId); + //this is a bit weird... we have to add the delay of the final packet + //we are actually receiving the HEAD bit of the request right now + sendDelayedExecutionEvent(req->bytes * channel_byte_delay_, f.callback); + flows_.erase(freq->flowId); + delete freq; + } else { + debug("Receive %" PRIu32 " bytes of flow %" PRIu64 ": %" PRIu64 " bytes left", + req->bytes, freq->flowId, f.bytesLeft); + f.bytesLeft -= flow_mtu_; //we have constructed so that the bytes left is always a multiple + freq->bytes = flow_mtu_; + auto* ev = newCallback(this, &MemoryModel::accessRequest, flow_rsp_id_, req); + TimeDelta delay = flow_mtu_ * f.byteRequestDelay; + sendDelayedExecutionEvent(delay, ev); + } +} + +void +SnapprMemoryModel::accessRequest(int linkId, Request *req) +{ + req->rspId = linkId; + ChannelQueue& q = channels_[channelInterleaver_]; + q.next_free = std::max(q.next_free, now()); + TimeDelta timeToSend = q.byte_delay * req->bytes; + //we send a response when the HEAD of the request is available + sendExecutionEvent(q.next_free, newCallback(rsp_handlers_[linkId], &RequestHandlerBase::handle, req)); + q.next_free += timeToSend; + debug("Channel %d busy with %d until %10.5e after delay of %10.5e", + channelInterleaver_, linkId, q.next_free.sec(), timeToSend.sec()); + channelInterleaver_ = (channelInterleaver_ + 1) % channels_.size(); +} + +} // end namespace ComputeLibrary +} // end namespace Hg +} // end namespace SST diff --git a/src/sst/elements/mercury/libraries/compute/memory_model.h b/src/sst/elements/mercury/libraries/compute/memory_model.h new file mode 100644 index 0000000000..8e07f3a25e --- /dev/null +++ b/src/sst/elements/mercury/libraries/compute/memory_model.h @@ -0,0 +1,68 @@ +// Copyright 2009-2024 NTESS. Under the terms +// of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Copyright (c) 2009-2024, NTESS +// All rights reserved. +// +// Portions are copyright of other developers: +// See the file CONTRIBUTORS.TXT in the top level directory +// of the distribution for more information. +// +// This file is part of the SST software package. For license +// information, see the LICENSE file in the top level directory of the +// distribution. + +#pragma once + +#include + +namespace SST { +namespace Hg { +namespace ComputeLibrary { + +class MemoryModel { +public: + MemoryModel(uint32_t id, SST::Params ¶ms, Node *parent); + + ~MemoryModel() override {} + + std::string toString() const override { return "packet flow memory model"; } + + void accessFlow(uint64_t bytes, TimeDelta min_byte_delay, + Callback *cb) override; + + void accessRequest(int linkId, Request *req) override; + + void flowRequestResponse(Request *req); + +private: + struct FlowRequest : public Request { + uint32_t flowId; + }; + + struct Flow { + ExecutionEvent *callback; + uint64_t bytesLeft; + TimeDelta byteRequestDelay; + }; + + struct ChannelQueue { + TimeDelta byte_delay; + std::vector reqs; + Timestamp next_free; + }; + + TimeDelta channel_byte_delay_; + std::unordered_map flows_; + std::vector channels_; + uint32_t flowId_; + uint32_t channelInterleaver_; + uint32_t mtu_; + uint32_t flow_mtu_; + int flow_rsp_id_; +}; + +} // end namespace ComputeLibrary +} // end namespace Hg +} // end namespace SST diff --git a/src/sst/elements/mercury/operating_system/process/thread.h b/src/sst/elements/mercury/operating_system/process/thread.h index 5964e7fc4f..d415eb035b 100644 --- a/src/sst/elements/mercury/operating_system/process/thread.h +++ b/src/sst/elements/mercury/operating_system/process/thread.h @@ -266,8 +266,8 @@ class Thread return active_cores_.size(); } - void computeDetailed(uint64_t flops, uint64_t intops, - uint64_t bytes, int nthread=use_omp_num_threads); + // void computeDetailed(uint64_t flops, uint64_t intops, + // uint64_t bytes, int nthread=use_omp_num_threads); void* getTlsValue(long thekey) const; diff --git a/src/sst/elements/mercury/pymercury.py b/src/sst/elements/mercury/pymercury.py index cf7288569f..0f58107e88 100644 --- a/src/sst/elements/mercury/pymercury.py +++ b/src/sst/elements/mercury/pymercury.py @@ -56,7 +56,7 @@ def __init__(self): def build(self,nid,lid,nranks): if self._check_first_build(): sst.addGlobalParams("params_%s"%self._instance_name, self._getGroupParams("params")) - node = sst.Component("node" + str(nid), "hg.node") + node = sst.Component("node" + str(nid), "hg.Node") node.addGlobalParamSet("params_%s"%self._instance_name) node.addParam("nodeID", nid) node.addParam("logicalID", lid)