detailed model needs a miranda cpu for both read and write DMAs

src/sst/elements/ember/embermemoryev.h

@@ -34,7 +34,7 @@ class EmberMemAllocEvent : public EmberEvent {
 
     void issue( uint64_t time, FOO* functor ) {
 
-        m_output->verbose(CALL_INFO, 2, 0, "length=%" PRIu64 "\n", m_length );
+        m_output->verbose(CALL_INFO, 2, 0, "length=%zu\n", m_length );
         EmberEvent::issue( time );
 
         std::function<void(uint64_t)> callback = [=](uint64_t value){

src/sst/elements/ember/mpi/embermpigen.h

@@ -289,7 +289,7 @@ void EmberMessagePassingGenerator::enQ_send( Queue& q,
     uint32_t count, PayloadDataType dtype, RankID dest, uint32_t tag,
     Communicator group)
 {
-	verbose(CALL_INFO,2,0,"payload=%p dest=%d tag=%#x\n",payload, dest, tag);
+	verbose(CALL_INFO,2,0,"payload=0x%" PRIx64 " dest=%d tag=%#x\n",(uint64_t) &payload, dest, tag);
     q.push( new EmberSendEvent( *cast(m_api), &getOutput(), m_Stats[Send],
 		payload, count, dtype, dest, tag, group ) );
 
@@ -321,7 +321,7 @@ void EmberMessagePassingGenerator::enQ_isend( Queue& q,
     uint32_t count, PayloadDataType dtype, RankID dest, uint32_t tag, 
     Communicator group, MessageRequest* req )
 {
-	verbose(CALL_INFO,2,0,"payload=%p dest=%d tag=%#x req=%p\n",payload, dest, tag, req );
+	verbose(CALL_INFO,2,0,"payload=0x%" PRIx64" dest=%d tag=%#x req=%p\n",(uint64_t)&payload, dest, tag, req );
     q.push( new EmberISendEvent( *cast(m_api), &getOutput(), m_Stats[Isend],
         payload, count, dtype, dest, tag, group, req ) );
 

src/sst/elements/ember/mpi/motifs/emberdetailedring.cc

@@ -24,6 +24,7 @@ EmberDetailedRingGenerator::EmberDetailedRingGenerator(SST::Component* owner, Pa
 	m_messageSize = params.find<uint32_t>("arg.messagesize", 1024);
 	m_iterations = params.find<int32_t>("arg.iterations", 1);
 	m_stream_n = params.find<int32_t>("arg.stream_n", 1000);
+	m_printRank = params.find<int32_t>("arg.printRank", 0);
 }
 
 std::string EmberDetailedRingGenerator::getComputeModelName()
@@ -40,9 +41,24 @@ inline long mod( long a, long b )
 bool EmberDetailedRingGenerator::generate( std::queue<EmberEvent*>& evQ) 
 {
    if ( m_loopIndex == m_iterations ) {
-        if ( 0 == rank()) {
+        if ( m_printRank == rank() || -1 == m_printRank ) {
+            double totalTime = (double)(m_stopTime - m_startTime)/1000000000.0;
+
+            double latency = ((totalTime/m_iterations)/size());
+            double bandwidth = (double) m_messageSize / latency;
+
+            output("%s total time %.3f us, loop %d, bufLen %d"
+                    ", latency %.3f us. bandwidth %f GB/s\n",
+                                getMotifName().c_str(),
+                                totalTime * 1000000.0, m_iterations,
+                                m_messageSize,
+                                latency * 1000000.0,
+                                bandwidth / 1000000000.0 );
+
             double computeTime = (double)(m_stopCompute - m_startCompute)/1000000000.0;
-            output("%s total compute %.3f sec\n", getMotifName().c_str(),computeTime * 1000000.0 );
+
+            output("%s total compute %.3f us\n", getMotifName().c_str(),
+                                computeTime * 1000000.0 );
         }
         return true;
     }
@@ -65,6 +81,12 @@ bool EmberDetailedRingGenerator::generate( std::queue<EmberEvent*>& evQ)
     int from = mod( (signed int) rank() - 1, size() );
     verbose( CALL_INFO, 2, 0, "to=%d from=%d\n",to,from);
 
+    if ( 0 == m_loopIndex ) {
+        verbose( CALL_INFO, 1, 0, "rank=%d size=%d\n", rank(), size());
+
+        enQ_getTime( evQ, &m_startTime );
+    }
+
     if ( 0 == rank() ) {
         enQ_isend( evQ, m_sendBuf, m_messageSize, CHAR, to, TAG,
                                                 GroupWorld, &m_req[0] );
@@ -114,16 +136,16 @@ bool EmberDetailedRingGenerator::generate( std::queue<EmberEvent*>& evQ)
         params.insert( "generatorParams.verbose", "1" );
         params.insert( "verbose", "1" );
 
-
         enQ_getTime( evQ, &m_startCompute );
         enQ_detailedCompute( evQ, motif, params );
         enQ_getTime( evQ, &m_stopCompute );
-
     }
 
 	enQ_wait( evQ, &m_req[0] );
 	enQ_wait( evQ, &m_req[1] );
 
-    ++m_loopIndex;
+    if ( ++m_loopIndex == m_iterations ) {
+        enQ_getTime( evQ, &m_stopTime );
+    }
     return false;
 }

src/sst/elements/ember/mpi/motifs/emberdetailedring.h

@@ -31,10 +31,13 @@ class EmberDetailedRingGenerator : public EmberMessagePassingGenerator {
 	uint32_t m_iterations;
     uint32_t m_loopIndex;
 	uint32_t m_stream_n;
+	uint32_t m_printRank;
     MessageResponse m_resp;
     Hermes::MemAddr    m_sendBuf;
     Hermes::MemAddr    m_recvBuf;
     Hermes::MemAddr    m_streamBuf;
+    uint64_t m_startTime;
+    uint64_t m_stopTime;
     uint64_t m_startCompute;
     uint64_t m_stopCompute;
 };

src/sst/elements/ember/mpi/motifs/emberring.h

@@ -25,6 +25,7 @@ class EmberRingGenerator : public EmberMessagePassingGenerator {
     bool generate( std::queue<EmberEvent*>& evQ);
 
 private:
+    MessageRequest  m_req[2];
 	uint32_t m_messageSize;
 	uint32_t m_iterations;
     uint32_t m_loopIndex;

src/sst/elements/ember/run/configurations/basicDetailedModel.py

@@ -19,7 +19,6 @@ class BasicDetailedModel(DetailedModel):
     def __init__(self, params ):
         self.name = 'BasicDetailedModel' 
         self.params = params
-        self.links = []
 
     def getName(self):
         return self.name
@@ -77,6 +76,7 @@ def build(self,nodeID,ransPerNode):
         #print 'BasicDetailedModel.build( nodeID={0}, ransPerNode={1} )'.format( nodeID, ransPerNode ) 
 
         self.links = []
+        self.nicLinks = []
 
         prefix = "basicModel_node" + str(nodeID) + "_"
 
@@ -104,8 +104,11 @@ def build(self,nodeID,ransPerNode):
                                     self.params['cpu_params'], \
                                     self.params['l1_params']  ) )
 
-        self.nicLink = self._createNic( prefix, bus, self.params['nic_cpu_params'],\
-                                    self.params['nic_l1_params'] )
+        self.nicLinks.append( self._createNic( prefix + 'read', bus, self.params['nic_cpu_params'],\
+                                    self.params['nic_l1_params'] ) )
+
+        self.nicLinks.append( self._createNic( prefix + 'write', bus, self.params['nic_cpu_params'],\
+                                    self.params['nic_l1_params'] ) )
 
         return True 
 
@@ -115,7 +118,7 @@ def getThreadLinks(self,core):
 
     def getNicLink(self ):
         #print 'BasicDetailedModel.getNicLink()' 
-        return self.nicLink 
+        return self.nicLinks[0], self.nicLinks[1] 
 
 def getModel(params):
     return BasicDetailedModel(params)

src/sst/elements/ember/run/configurations/detailedSim.py

@@ -21,13 +21,15 @@
 topo = 'torus'
 shape = '2'
 
-detailedNodes = [0]
+detailedNodes = [1]
 
 detailedModel = "basicDetailedModel" 
 detailedModelParams = "basicDetailedModelParams" 
 
-xxx = "Ring n=100"
-yyy = "DetailedRing n=100"
+arguments = 'messagesize=80000 printRank=-1'
+
+xxx = "Ring computeTime=0 " + arguments
+yyy = "DetailedRing " + arguments
 
 def genWorkFlow( defaults, nodeNum = None ):
 

src/sst/elements/ember/run/configurations/sandyBridgeModel.py

@@ -77,12 +77,13 @@ def build(self,nodeID,ranksPerNode):
 
         prefix = "sandyBridgeModel_node" + str(nodeID) + "_"
 
-        cpuL1s, nicL1 = snb.configure(prefix,self.params)
+        cpuL1s, nicL1_read, nicL1_write = snb.configure(prefix,self.params)
 
         self.links.append( self._createThreads( prefix, cpuL1s, self.params['cpu_params']  ) )
 
-
-        self.nicLink = self._createNic( prefix, nicL1, self.params['nic_cpu_params']) 
+        self.nicLink = []    
+        self.nicLink.append( self._createNic( prefix + 'read', nicL1_read, self.params['nic_cpu_params']) ) 
+        self.nicLink.append( self._createNic( prefix + 'write', nicL1_write, self.params['nic_cpu_params']) )
 
         return True 
 
@@ -92,7 +93,7 @@ def getThreadLinks(self,nodeRank):
 
     def getNicLink(self ):
         #print 'SandyBridgeModel.getNicLink()' 
-        return self.nicLink 
+        return self.nicLink[0], self.nicLink[1] 
 
 def getModel(params):
     return SandyBridgeModel(params)

src/sst/elements/ember/run/configurations/sandyBridgeModelParams.py

@@ -35,7 +35,7 @@
 memory_capacity = 16384     # Size of memory in MBs
 
 
-num_routers = groups * (cores_per_group + memory_controllers_per_group + l3cache_blocks_per_group) + 1
+num_routers = groups * (cores_per_group + memory_controllers_per_group + l3cache_blocks_per_group) + 2 
 
 l1_prefetch_params = {
 }

src/sst/elements/ember/run/configurations/sandySim.py

@@ -26,8 +26,10 @@
 detailedModel = "sandyBridgeModel" 
 detailedModelParams = "sandyBridgeModelParams" 
 
-xxx = "Ring n=100"
-yyy = "DetailedRing n=100"
+arguments = 'messagesize=100000 printRank=-1' 
+
+xxx = "Ring computeTime=10998000 " + arguments 
+yyy = "DetailedRing " + arguments 
 
 def genWorkFlow( defaults, nodeNum = None ):
 

src/sst/elements/ember/run/configurations/snb.py

@@ -188,11 +188,16 @@ def _configCache( prefix,
 
     print 'Creating NIC L1/L2'
 
-    nicL1 = _configureL1L2( prefix, 'nic', l1_params, l1_prefetch_params,
+    nicL1_read = _configureL1L2( prefix, 'nic_read', l1_params, l1_prefetch_params,
+                                l2_params, l2_prefetch_params, next_network_id, ring_latency,
+                                router_map[prefix + "rtr." + str(next_network_id)])
+
+    next_network_id = next_network_id + 1
+    nicL1_write = _configureL1L2( prefix, 'nic_write', l1_params, l1_prefetch_params,
                                 l2_params, l2_prefetch_params, next_network_id, ring_latency,
                                 router_map[prefix + "rtr." + str(next_network_id)])
 
-    return cpuL1s, nicL1
+    return cpuL1s, nicL1_read, nicL1_write
 
 
 def configure( prefix, params ):

src/sst/elements/ember/run/lib/emberEP.py

@@ -51,7 +51,9 @@ def build( self, nodeID, extraKeys ):
 
         if detailed:
             detailed.build(nodeID,ranksPerNode)
-            nic.addLink( detailed.getNicLink( ), "detailed0", "1ps" )
+            read, write = detailed.getNicLink( )
+            nic.addLink( read, "detailed0", "1ps" )
+            nic.addLink( write, "detailed1", "1ps" )
 
             memory = sst.Component("memory" + str(nodeID), "thornhill.MemoryHeap")
             memory.addParam( "nid", nodeID )

src/sst/elements/firefly/nic.cc

@@ -30,7 +30,7 @@ Nic::Nic(ComponentId_t id, Params &params) :
     m_sendNotifyCnt(0),
     m_sendMachine( 2, SendMachine( *this, m_dbg ) ),
     m_recvMachine( *this, m_dbg ),
-    m_detailedCompute(NULL),
+    m_detailedCompute( 2, NULL ),
     m_getKey(10)
 {
     m_myNodeId = params.find<int>("nid", -1);
@@ -111,14 +111,29 @@ Nic::Nic(ComponentId_t id, Params &params) :
 
     if ( ! dtldName.empty() ) {
 
-        m_detailedCompute = dynamic_cast<Thornhill::DetailedCompute*>( loadSubComponent(
+        dtldParams.insert( "portName", "read", true );
+        Thornhill::DetailedCompute* detailed;
+        detailed = dynamic_cast<Thornhill::DetailedCompute*>( loadSubComponent(
                             dtldName, this, dtldParams ) );
 
-        assert( m_detailedCompute );
-        if ( ! m_detailedCompute->isConnected() ) {
+        assert( detailed );
+        if ( ! detailed->isConnected() ) {
 
-            delete m_detailedCompute;
-            m_detailedCompute = NULL;
+            delete detailed;
+        } else {
+            m_detailedCompute[0] = detailed;
+        }
+
+        dtldParams.insert( "portName", "write", true );
+        detailed = dynamic_cast<Thornhill::DetailedCompute*>( loadSubComponent(
+                            dtldName, this, dtldParams ) );
+
+        assert( detailed );
+        if ( ! detailed->isConnected() ) {
+
+            delete detailed;
+        } else {
+            m_detailedCompute[1] = detailed;
         }
     }
 }

src/sst/elements/firefly/nic.h

@@ -529,8 +529,8 @@ class Nic : public SST::Component  {
     int getNum_vNics() { return m_num_vNics; }
     void printStatus(Output &out);
 
-    void detailedMemOp( std::vector<DmaVec>& vec, std::string op,
-													Callback callback ) {
+    void detailedMemOp( Thornhill::DetailedCompute* detailed,
+            std::vector<DmaVec>& vec, std::string op, Callback callback ) {
 
         std::deque< std::pair< std::string, SST::Params> > gens;
         m_dbg.verbose(CALL_INFO,1,NIC_DBG_DETAILED_MEM,
@@ -581,7 +581,7 @@ class Nic : public SST::Component  {
 			params.insert( "memOp", op );		
 			#if 0
 			params.insert( "generatorParams.verbose", "1" );		
-			params.insert( "verbose", "1" );		
+			params.insert( "verbose", "5" );		
 			#endif
 
         	gens.push_back( std::make_pair( "miranda.SingleStreamGenerator", params ) );
@@ -591,19 +591,20 @@ class Nic : public SST::Component  {
 			schedCallback( callback, 0 );
 		} else {
 	    	std::function<int()> foo = [=](){
+                printf("mem op done\n");
            		callback( );
 				return 0;
 			};
-			m_detailedCompute->start( gens, foo );	
+			detailed->start( gens, foo );	
 		}
 
 	}
 
     void dmaRead( std::vector<DmaVec>& vec, Callback callback ) {
 
-		if ( m_detailedCompute ) {
+		if ( m_detailedCompute[0] ) {
 
-			detailedMemOp( vec, "Read", callback );
+			detailedMemOp( m_detailedCompute[0], vec, "Read", callback );
 
 		} else {
 			size_t len = 0;	
@@ -616,9 +617,9 @@ class Nic : public SST::Component  {
 
     void dmaWrite( std::vector<DmaVec>& vec, Callback callback ) {
 
-		if ( m_detailedCompute ) {
+		if ( m_detailedCompute[1] ) {
 
-			detailedMemOp( vec, "Write", callback );
+			detailedMemOp( m_detailedCompute[1], vec, "Write", callback );
 
 		} else {
 			size_t len = 0;	
@@ -719,7 +720,7 @@ class Nic : public SST::Component  {
 
     Output                  m_dbg;
     std::vector<VirtNic*>   m_vNicV;
-	Thornhill::DetailedCompute* m_detailedCompute;
+    std::vector<Thornhill::DetailedCompute*> m_detailedCompute;
 
     uint16_t m_getKey;
   public: