add: performance improvement for large data exports

AlexanderFroemmgen · Mar 20, 2018 · 7b50145 · 7b50145
1 parent db127ea
commit 7b50145
Showing 1 changed file with 8 additions and 6 deletions.
diff --git a/maci_backend/Controllers/ExperimentDataExportController.cs b/maci_backend/Controllers/ExperimentDataExportController.cs
@@ -103,6 +103,8 @@ private void GetResultDataCSVExport(int id, string path)
                     .Include(s => s.ExperimentInstances)
                     .Single(s => s.Id == id);
 
+            var outputFileStream = new StreamWriter(new FileStream(path, FileMode.Create));
+
             var result = new StringBuilder("simInstanceId;");
 
             foreach (var parameter in data.Parameters.OrderBy(p => p.Name))
@@ -111,12 +113,12 @@ private void GetResultDataCSVExport(int id, string path)
                 result.Append(";");
             }
             result.Append("value;offset;key;key2;key3;\n");
-            System.IO.File.WriteAllText(path, result.ToString());
+            outputFileStream.Write(result);            
             result.Clear();
 
             int counter = -1;
             int total = data.ExperimentInstances.Count();
-                var sb = new StringBuilder();
+            var sb = new StringBuilder();
 
             foreach (var simInstanceBlank in data.ExperimentInstances)
             {
@@ -190,15 +192,15 @@ private void GetResultDataCSVExport(int id, string path)
                 /* end denny */
 
                 counter++;
-                if (counter % 100 == 0)
+                if (counter % 50 == 0)
                 {
                     Console.WriteLine("Exported " + counter.ToString() + " of " + total.ToString());
-                    System.IO.File.AppendAllText(path, sb.ToString());
-sb.Clear();
+                    outputFileStream.Write(sb);
+                    sb.Clear();
                 }
             }
             /* write remaining stuff */
-            System.IO.File.AppendAllText(path, sb.ToString());
+            outputFileStream.Write(sb);
             Console.WriteLine("Finished Export");
         }