Check and report cycles

Signed-off-by: Chen Wang <wangvsa@gmail.com>

tools/verifyio/match_mpi.py

@@ -115,7 +115,7 @@ def read_one_mpi_call(self, rank, seq_id, record):
         # str first
         args = []
         for i in range(record.arg_count):
-            arg = record.args[i].decode("utf-8")
+            arg = record.args[i].decode("utf-8", "ignore")
             args.append(arg)
 
         src, dst, stag, rtag = None, None, None, None
@@ -433,7 +433,7 @@ def add_nodes_to_edge(edge, call):
         coll_call.matched = True
 
     mpi_call.matched = True
-    #print("match collective:", mpi_call.func, "root:", mpi_call.src, mpi_call.comm)
+    #print("match collective:", mpi_call.func, "root:", mpi_call.src, mpi_call.comm, mpi_call.req, mpi_call.reqflag)
     return edge
 
 #@profile

tools/verifyio/verifyio.py

@@ -260,11 +260,16 @@ def get_conflict_info(nodes: list, reader: RecorderReader, summary=None, show_de
 
     t1 = time.time()
     G = VerifyIOGraph(all_nodes, mpi_edges, include_vc=True)
-    G.run_vector_clock()
-    #G.run_transitive_closure()
     t2 = time.time()
     print("build happens-before graph: %.3f secs, nodes: %d" %((t2-t1), G.num_nodes()))
 
+    # Correct code (traces) should result in 
+    # a DAG without any cycles
+    if G.check_cycles(): quit()
+
+    G.run_vector_clock()
+    #G.run_transitive_closure()
+
     # G.plot_graph("vgraph.jpg")
 
     t1 = time.time()

tools/verifyio/verifyio_graph.py

@@ -74,7 +74,7 @@ def next_hb_node(self, current, funcs, target_rank):
         nodes = self.nodes[target_rank]
         for target in nodes:
             if (target.func in funcs) and (self.has_path(current, target)):
-                targest.append(target)
+                target.append(target)
                 break
         return target
 
@@ -173,7 +173,7 @@ def __build_graph(self, all_nodes, mpi_edges, include_vc):
         # have added all nodes. We use this function
         # to add edges of matching MPI calls
         ghost_node_index = 0
-        for edge in mpi_edges:
+        for edge in mpi_edges[0:161]:
             head, tail = edge.head, edge.tail
             # all-to-all
             # TODO is there a many-to-many MPI call that
@@ -214,6 +214,7 @@ def __build_graph(self, all_nodes, mpi_edges, include_vc):
 
             # many-to-one, e.g., reduce
             elif edge.call_type == MPICallType.MANY_TO_ONE:
+                print(head, tail)
                 for h in head:
                     self.add_edge(h, tail)
             # one-to-many, e.g., bcast
@@ -229,3 +230,25 @@ def __build_graph(self, all_nodes, mpi_edges, include_vc):
         #print("Generate transitive closure")
         #tc = nx.transitive_closure(self.G)
 
+    # Detect cycles of the graph
+    # correct code should contain no cycles.
+    # incorrect code, e.g., with unmatched collective calls
+    # may have cycles
+    # e.g., pnetcdf/test/testcases/test-varm.c
+    # it uses ncmpi_wait() call, which may result in 
+    # rank0 calling MPI_File_write_at_all(), and
+    # rank1 calling MPI_File_write_all()
+    # 
+    # Our verification algorithm assumes the graph
+    # has no code, so we need do this check first.
+    def check_cycles(self):
+        has_cycles = False
+        try:
+            cycle = nx.find_cycle(self.G)
+            print("Generated graph contain cycles. Original code may have bugs.")
+            print(cycle)
+            has_cycles = True
+        except nx.exception.NetworkXNoCycle:
+            pass
+        return has_cycles
+