We only store 25 error reports now for each incident, but keep track of volume to be able to determine impact of each incident.

Author: jgauffin

src/Server/Coderr.Server.Api.Client/ServerApiClient.cs

@@ -106,8 +106,15 @@ public void Open(Uri uri, string apiKey, string sharedSecret)
         private async Task<TResult> DeserializeResponse<TResult>(HttpContent content)
         {
             var jsonStr = await content.ReadAsStringAsync();
-            var responseObj = JsonConvert.DeserializeObject(jsonStr, typeof(TResult), _jsonSerializerSettings);
-            return (TResult)responseObj;
+            try
+            {
+                var responseObj = JsonConvert.DeserializeObject(jsonStr, typeof(TResult), _jsonSerializerSettings);
+                return (TResult) responseObj;
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException("Failed to deserialize " + jsonStr, ex);
+            }
         }
 
         private async Task<HttpResponseMessage> RequestAsync(HttpMethod httpMethod, string cqsType, object cqsObject)

src/Server/Coderr.Server.App/Core/Reports/Jobs/DeleteOldReports.cs

@@ -174,8 +174,10 @@ private async Task<AppResult[]> GetReportCounts(DateTime lastMonth)
             using (var cmd = _unitOfWork.CreateDbCommand())
             {
                 cmd.CommandText = @"SELECT ApplicationId, count(*)
-                                    FROM ErrorReports
-                                    where CreatedAtUtc >= @fromDate AND CreatedAtUtc < @toDate
+                                    FROM IncidentReports
+                                    JOIN Incidents ON (Incidents.Id = IncidentId)
+                                    where IncidentReports.ReceivedAtUtc >= @fromDate 
+                                        AND IncidentReports.ReceivedAtUtc < @toDate
                                     group by ApplicationId";
                 cmd.AddParameter("fromDate", lastMonth);
                 cmd.AddParameter("toDate", lastMonth.AddMonths(1));

src/Server/Coderr.Server.App/Modules/MonthlyStats/CollectStatsJob.cs

@@ -15,7 +15,7 @@ public interface IReportsRepository
         /// <param name="errorId">Customer generated id (from the client library).</param>
         /// <returns>report if found; otherwise <c>null</c>.</returns>
         /// <exception cref="ArgumentNullException">errorId</exception>
-        Task<ErrorReportEntity> FindByErrorIdAsync(string errorId);
+        Task<ReportMapping> FindByErrorIdAsync(string errorId);
 
 
         /// <summary>

src/Server/Coderr.Server.Domain/Core/ErrorReports/IReportsRepository.cs

@@ -0,0 +1,38 @@
+using System;
+
+namespace Coderr.Server.Domain.Core.ErrorReports
+{
+    /// <summary>
+    ///     Maps a received report to an incident
+    /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         Since we do not store complete reports for everything any more. Allows us to still keep
+    ///         track of client side generated error ids and which incident they belong to.
+    ///     </para>
+    /// </remarks>
+    public class ReportMapping
+    {
+        public int Id { get; set; }
+
+        /// <summary>
+        /// Incident that the report belongs to.
+        /// </summary>
+        public int IncidentId { get; set; }
+
+        /// <summary>
+        /// Client report id
+        /// </summary>
+        public string ErrorId { get; set; }
+
+        /// <summary>
+        /// Only when fetching items
+        /// </summary>
+        public int ApplicationId { get; set; }
+
+        /// <summary>
+        /// When the report was generated in the client
+        /// </summary>
+        public DateTime ReceivedAtUtc { get; set; }
+    }
+}

src/Server/Coderr.Server.Domain/Core/ErrorReports/ReportMapping.cs

@@ -26,13 +26,13 @@ public async Task HandleAsync(IMessageContext context, ReportAddedToIncident e)
         {
             try
             {
-                _logger.Debug("storing feedback for report " + e.Report.ReportId);
                 var userInfo = Enumerable.FirstOrDefault(e.Report.ContextCollections, x => x.Name == "UserSuppliedInformation");
                 if (userInfo == null)
                     return;
 
                 userInfo.Properties.TryGetValue("Description", out var description);
                 userInfo.Properties.TryGetValue("Email", out var email);
+                _logger.Debug($"storing feedback for report {e.Report.ReportId}: {email} {description}");
                 var cmd = new SubmitFeedback(e.Report.ReportId, e.Report.RemoteAddress ?? "")
                 {
                     Feedback = description,

src/Server/Coderr.Server.ReportAnalyzer/Feedback/Handlers/StoreFeedbackFromNewReports.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading.Tasks;
 using Coderr.Server.Domain.Core.ErrorReports;
 using Coderr.Server.ReportAnalyzer.Incidents;
 
@@ -71,5 +72,6 @@ public interface IAnalyticsRepository
         int GetMonthReportCount();
 
         void AddMissedReport(DateTime date);
+        Task StoreReportStats(ReportMapping mapping);
     }
 }

src/Server/Coderr.Server.ReportAnalyzer/IAnalyticsRepository.cs

@@ -102,11 +102,12 @@ public async Task Analyze(IMessageContext context, ErrorReportEntity report)
             }
             else
             {
-                if (incident.ReportCount > 1000)
+                await _repository.StoreReportStats(new ReportMapping()
                 {
-                    _logger.Debug("Report count is more than 10000. Ignoring report for incident " + incident.Id);
-                    return;
-                }
+                    IncidentId = incident.Id,
+                    ErrorId = report.ClientReportId,
+                    ReceivedAtUtc = report.CreatedAtUtc
+                });
 
                 if (incident.IsIgnored)
                 {
@@ -139,6 +140,11 @@ public async Task Analyze(IMessageContext context, ErrorReportEntity report)
 
                 incident.AddReport(report);
                 _repository.UpdateIncident(incident);
+                if (incident.ReportCount > 25)
+                {
+                    _logger.Debug("Report count is more than 25. Storing only report stats for incident " + incident.Id);
+                    return;
+                }
             }
 
             if (!string.IsNullOrWhiteSpace(report.EnvironmentName))

src/Server/Coderr.Server.ReportAnalyzer/Inbound/Handlers/Reports/ReportAnalyzer.cs

@@ -25,4 +25,8 @@
   <ItemGroup>
     <EmbeddedResource Include="Schema\*.sql" />
   </ItemGroup>
+
+  <ItemGroup>
+    <None Remove="Schema\Coderr.v19.sql" />
+  </ItemGroup>
 </Project>

src/Server/Coderr.Server.SqlServer/Coderr.Server.SqlServer.csproj

@@ -48,7 +48,7 @@ public async Task<GetApplicationOverviewResult> HandleAsync(IMessageContext cont
                         new { version = query.Version });
                     filter1 = @"JOIN IncidentVersions On (Incidents.Id = IncidentVersions.IncidentId)
                             WHERE IncidentVersions.VersionId = @versionId AND ";
-                    filter2 = @"JOIN IncidentVersions On (ErrorReports.IncidentId = IncidentVersions.IncidentId)
+                    filter2 = @"JOIN IncidentVersions On (IncidentReports.IncidentId = IncidentVersions.IncidentId)
                             WHERE IncidentVersions.VersionId = @versionId AND ";
                     cmd.AddParameter("versionId", id);
                 }
@@ -63,18 +63,19 @@ from Incidents
 AND Incidents.CreatedAtUtc <= GetUtcDate()
 {0}
 group by cast(Incidents.CreatedAtUtc as date);
-select cast(ErrorReports.CreatedAtUtc as date), count(Id)
-from ErrorReports
-{3} ErrorReports.CreatedAtUtc >= @minDate
-AND ErrorReports.CreatedAtUtc <= GetUtcDate()
+select cast(IncidentReports.CreatedAtUtc as date), count(Id)
+from IncidentReports
+join Incidents isa ON (isa.Id = IncidentReports.IncidentId)
+{3} IncidentReports.ReceivedAtUtc >= @minDate
+AND IncidentReports.ReceivedAtUtc <= GetUtcDate()
 {1}
-group by cast(ErrorReports.CreatedAtUtc as date);";
+group by cast(IncidentReports.ReceivedAtUtc as date);";
 
                 if (query.ApplicationId > 0)
                 {
                     cmd.CommandText = string.Format(sql,
                         " AND Incidents.ApplicationId = @appId",
-                        " AND ErrorReports.ApplicationId = @appId",
+                        " AND isa.ApplicationId = @appId",
                         filter1, filter2);
                     cmd.AddParameter("appId", query.ApplicationId);
                 }
@@ -127,11 +128,11 @@ JOIN IncidentVersions ON (Incidents.Id = IncidentVersions.IncidentId)
 AND Incidents.State <> {(int)IncidentState.Ignored}
 AND Incidents.State <> {(int)IncidentState.Closed};
 
-SELECT count(id) from ErrorReports 
-JOIN IncidentVersions ON (ErrorReports.IncidentId = IncidentVersions.IncidentId)
+SELECT count(id) from IncidentReports
+JOIN IncidentVersions ON (IncidentReports.IncidentId = IncidentVersions.IncidentId)
 WHERE IncidentVersions.VersionId = @versionId
-AND CreatedAtUtc >= @minDate
-AND CreatedAtUtc <= GetUtcDate()
+AND ReceivedAtUtc >= @minDate
+AND ReceivedAtUtc <= GetUtcDate()
 AND ApplicationId = @appId;
 
 SELECT count(distinct emailaddress) from IncidentFeedback
@@ -162,9 +163,11 @@ AND Description is not null
 AND Incidents.State <> {(int)IncidentState.Ignored}
 AND Incidents.State <> {(int)IncidentState.Closed};
 
-select count(id) from ErrorReports 
-where CreatedAtUtc >= @minDate
-AND CreatedAtUtc <= GetUtcDate()
+SELECT count(id) 
+FROM IncidentReports
+JOIN Incidents ON (Incidents.Id = IncidentReports.IncidentId)
+WHERE ReceivedAtUtc >= @minDate
+AND ReceivedAtUtc <= GetUtcDate()
 AND ApplicationId = @appId;
 
 select count(distinct emailaddress) from IncidentFeedback
@@ -236,7 +239,7 @@ private async Task<GetApplicationOverviewResult> GetTodaysOverviewAsync(GetAppli
                         new { version = query.Version });
                     filter1 = @"JOIN IncidentVersions On (Incidents.Id = IncidentVersions.IncidentId)
                             WHERE IncidentVersions.VersionId = @versionId AND ";
-                    filter2 = @"JOIN IncidentVersions On (ErrorReports.IncidentId = IncidentVersions.IncidentId)
+                    filter2 = @"JOIN IncidentVersions On (IncidentReports.IncidentId = IncidentVersions.IncidentId)
                             WHERE IncidentVersions.VersionId = @versionId AND ";
                     cmd.AddParameter("versionId", id);
                 }
@@ -252,12 +255,13 @@ from Incidents
  AND Incidents.CreatedAtUtc <= GetUtcDate()
  AND Incidents.ApplicationId = @appId
  group by DATEPART(HOUR, Incidents.CreatedAtUtc);
- select DATEPART(HOUR, ErrorReports.CreatedAtUtc), cast(count(Id) as int)
- from ErrorReports
- {1} ErrorReports.CreatedAtUtc >= @minDate
- AND ErrorReports.CreatedAtUtc <= GetUtcDate()
- AND ErrorReports.ApplicationId = @appId
- group by DATEPART(HOUR, ErrorReports.CreatedAtUtc);";
+ select DATEPART(HOUR, IncidentReports.ReceivedAtUtc), cast(count(Id) as int)
+ from IncidentReports
+ JOIN Incidents ice ON (ice.Id = IncidentId)
+ {1} IncidentReports.ReceivedAtUtc >= @minDate
+ AND IncidentReports.ReceivedAtUtc <= GetUtcDate()
+ AND ice.ApplicationId = @appId
+ group by DATEPART(HOUR, IncidentReports.ReceivedAtUtc);";
 
                 cmd.CommandText = string.Format(sql, filter1, filter2);
                 cmd.AddParameter("appId", query.ApplicationId);