From 8ab7a25248badfb007ee1d40bd403e8997090176 Mon Sep 17 00:00:00 2001 From: yophilav <54859653+yophilav@users.noreply.github.com> Date: Thu, 17 Jun 2021 12:20:22 -0700 Subject: [PATCH] Use Docker Timestamp When Log Timestamp is not Available in JSON log. (#4981) Context: It generates the attached file, if a module does not use the prescribed logging format (which most module do not), the timestamp is 0. This makes the logs not very useful. The timestamp irrespective of logging format is actually available from Docker log API. Solution: This PR issues `timestamps` from edgeAgent via mgmt.sock to prepending the returning log with timestamp. edgeAgent then parse the timestamp & return the value in json format to portal. Note: This feature requires https://github.com/Azure/iotedge/pull/4970 --- builds/checkin/dotnet.yaml | 21 +++++- .../IRuntimeInfoProvider.cs | 2 +- .../logs/ILogMessageParser.cs | 1 + .../logs/LogMessageParser.cs | 15 +++- .../logs/LogsProvider.cs | 9 ++- .../logs/ModuleLogFilter.cs | 14 ++-- .../RuntimeInfoProvider.cs | 3 +- .../IModuleManager.cs | 2 +- .../ModuleManagementHttpClient.cs | 4 +- .../RuntimeInfoProvider.cs | 4 +- .../ModuleManagementHttpClientVersioned.cs | 6 +- .../blob/AzureBlobRequestsUploader.cs | 1 + .../KubernetesRuntimeInfoProvider.cs | 2 +- .../logs/LogsProcessorTest.cs | 24 +++---- .../logs/LogsProviderTest.cs | 71 ++++++++++++------- .../logs/LogsRequestToOptionsMapperTests.cs | 2 +- .../requests/LogsRequestHandlerTests.cs | 8 +-- .../requests/LogsUploadRequestHandlerTest.cs | 18 ++--- .../RuntimeInfoProviderTest.cs | 6 +- .../ModuleManagementHttpClientTest.cs | 2 +- .../RuntimeInfoProviderTest.cs | 4 +- .../stream/LogsStreamRequestHandlerTest.cs | 4 +- .../DummyModuleManager.cs | 2 +- .../EdgeAgentDirectMethods.cs | 6 +- 24 files changed, 148 insertions(+), 83 deletions(-) diff --git a/builds/checkin/dotnet.yaml b/builds/checkin/dotnet.yaml index 0127bdd9802..ce38f7fc2e2 100644 --- a/builds/checkin/dotnet.yaml +++ b/builds/checkin/dotnet.yaml @@ -45,7 +45,12 @@ jobs: displayName: Test inputs: filePath: scripts/linux/runTests.sh - arguments: '"--filter Category=Unit"' + arguments: '"--filter Category=Unit&Category!=GetLogsTests"' + - task: Bash@3 + displayName: Run GetLogsTests Tests + inputs: + filePath: scripts/linux/runTests.sh + arguments: '"--filter Category=GetLogsTests"' - task: PublishTestResults@2 displayName: Publish Test Results inputs: @@ -75,7 +80,19 @@ jobs: testSelector: testAssemblies testAssemblyVer2: | target\*Test.dll - testFiltercriteria: Category=Unit + testFiltercriteria: Category=Unit&Category!=GetLogsTests + runInParallel: true + runTestsInIsolation: true + codeCoverageEnabled: true + runSettingsFile: CodeCoverage.runsettings + publishRunAttachments: true + - task: VSTest@2 + displayName: Run GetLogsTests unit tests with code coverage + inputs: + testSelector: testAssemblies + testAssemblyVer2: | + target\*Test.dll + testFiltercriteria: Category=GetLogsTests runInParallel: true runTestsInIsolation: true codeCoverageEnabled: true diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs index c2288987186..455d707d7c5 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs @@ -16,7 +16,7 @@ public interface IRuntimeInfoProvider { Task> GetModules(CancellationToken cancellationToken); - Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken); + Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken); Task GetSystemInfo(CancellationToken cancellationToken); } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs index 5d32b21300e..df7481bc5ea 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs @@ -3,6 +3,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs { extern alias akka; using akka::Akka.IO; + using Microsoft.Azure.Devices.Edge.Util; public interface ILogMessageParser { diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs index 0e55cd40350..2a85b84befd 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs @@ -29,7 +29,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs public class LogMessageParser : ILogMessageParser { const int DefaultLogLevel = 6; - const string LogRegexPattern = @"^(<(?\d)>)?\s*((?\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d{3}\s[+-]\d{2}:\d{2})\s)?\s*(?.*)"; + const string LogRegexPattern = @"^((?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{9}Z))?\s*(<(?\d)>)?\s*((?\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d{3}\s[+-]\d{2}:\d{2})\s)?\s*(?.*)"; readonly string iotHubName; readonly string deviceId; @@ -48,6 +48,7 @@ internal static ModuleLogMessageData GetLogMessage(ByteString arg, string iotHub string stream = GetStream(arg[0]); ByteString payload = arg.Slice(8); string payloadString = payload.ToString(Encoding.UTF8); + (int logLevel, Option timeStamp, string logText) = ParseLogText(payloadString); var moduleLogMessage = new ModuleLogMessageData(iotHubName, deviceId, moduleId, stream, logLevel, timeStamp, logText, arg, payloadString); return moduleLogMessage; @@ -58,14 +59,24 @@ internal static ModuleLogMessageData GetLogMessage(ByteString arg, string iotHub internal static (int logLevel, Option timeStamp, string text) ParseLogText(string value) { var regex = new Regex(LogRegexPattern); + var match = regex.Match(value); int logLevel = DefaultLogLevel; string text = value; Option timeStamp = Option.None(); if (match.Success) { + var dtsg = match.Groups["dockerTimestamp"]; + if (dtsg?.Length > 0) + { + if (DateTime.TryParse(dtsg.Value, out DateTime dt)) + { + timeStamp = Option.Some(dt); + } + } + var tsg = match.Groups["timestamp"]; - if (tsg?.Length > 0) + if (tsg?.Length > 0 && !timeStamp.HasValue) { if (DateTime.TryParse(tsg.Value, out DateTime dt)) { diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs index 7678066c630..1533dcfd744 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs @@ -26,7 +26,12 @@ public async Task GetLogs(string id, ModuleLogOptions logOptions, Cancel { Preconditions.CheckNotNull(logOptions, nameof(logOptions)); - Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, false, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, cancellationToken); + if (logOptions.ContentType == LogsContentType.Json) + { + logOptions.Filter.IncludeTimestamp = Option.Some(true); + } + + Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, false, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, logOptions.Filter.IncludeTimestamp, cancellationToken); Events.ReceivedStream(id); byte[] logBytes = await this.GetProcessedLogs(id, logsStream, logOptions); @@ -60,7 +65,7 @@ internal async Task GetLogsStreamInternal(string id, ModuleLogOptions logOptions Preconditions.CheckNotNull(logOptions, nameof(logOptions)); Preconditions.CheckNotNull(callback, nameof(callback)); - Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, logOptions.Follow, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, cancellationToken); + Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, logOptions.Follow, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, logOptions.Filter.IncludeTimestamp, cancellationToken); Events.ReceivedStream(id); await this.logsProcessor.ProcessLogsStream(id, logsStream, logOptions, callback); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs index b88ddd3414a..ddbe9248b56 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs @@ -10,21 +10,22 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs public class ModuleLogFilter : IEquatable { - public static ModuleLogFilter Empty = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); + public static ModuleLogFilter Empty = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); - public ModuleLogFilter(Option tail, Option since, Option until, Option logLevel, Option regex) + public ModuleLogFilter(Option tail, Option since, Option until, Option logLevel, Option includeTimestamp, Option regex) { this.Tail = tail; this.Since = since; this.Until = until; this.LogLevel = logLevel; this.RegexString = regex; + this.IncludeTimestamp = includeTimestamp; this.Regex = regex.Map(r => new Regex(r)); } [JsonConstructor] - ModuleLogFilter(int? tail, string since, string until, int? loglevel, string regex) - : this(Option.Maybe(tail), Option.Maybe(since), Option.Maybe(until), Option.Maybe(loglevel), Option.Maybe(regex)) + ModuleLogFilter(int? tail, string since, string until, int? loglevel, bool? includeTimestamp, string regex) + : this(Option.Maybe(tail), Option.Maybe(since), Option.Maybe(until), Option.Maybe(loglevel), Option.Maybe(includeTimestamp), Option.Maybe(regex)) { } @@ -44,6 +45,9 @@ public ModuleLogFilter(Option tail, Option since, Option un [JsonConverter(typeof(OptionConverter), true)] public Option LogLevel { get; } + [JsonIgnore] + public Option IncludeTimestamp { get; set; } + [JsonIgnore] public Option Regex { get; } @@ -59,6 +63,7 @@ public bool Equals(ModuleLogFilter other) this.Tail.Equals(other.Tail) && this.Since.Equals(other.Since) && this.LogLevel.Equals(other.LogLevel) && + this.IncludeTimestamp.Equals(other.IncludeTimestamp) && this.RegexString.Equals(other.RegexString); public override int GetHashCode() @@ -67,6 +72,7 @@ public override int GetHashCode() hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.Tail); hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.Since); hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.LogLevel); + hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.IncludeTimestamp); hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.RegexString); return hashCode; } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs index 4bdaf1e6274..d360fd1f7b8 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs @@ -79,7 +79,7 @@ await Task.WhenAll( return modules; } - public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) + public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) { var containerLogsParameters = new ContainerLogsParameters { @@ -89,6 +89,7 @@ public Task GetModuleLogs(string module, bool follow, Option tail, }; tail.ForEach(t => containerLogsParameters.Tail = t.ToString()); since.ForEach(t => containerLogsParameters.Since = t.ToString()); + includeTimestamp.ForEach(t => containerLogsParameters.Timestamps = t); return this.client.Containers.GetContainerLogsAsync(module, containerLogsParameters, cancellationToken); } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs index 4c25467fccc..bc34d786c3f 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs @@ -33,7 +33,7 @@ public interface IModuleManager Task PrepareUpdateAsync(ModuleSpec moduleSpec); - Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken); + Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken); Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token); } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs index 58b4b59e4b9..9815d07d728 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs @@ -57,8 +57,8 @@ public ModuleManagementHttpClient(Uri managementUri, string serverSupportedApiVe public Task ReprovisionDeviceAsync() => this.inner.ReprovisionDeviceAsync(); - public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) => - this.inner.GetModuleLogs(name, follow, tail, since, until, cancellationToken); + public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) => + this.inner.GetModuleLogs(name, follow, tail, since, until, includeTimestamp, cancellationToken); public Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token) => this.inner.GetSupportBundle(since, until, iothubHostname, edgeRuntimeOnly, token); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs index fded4713e33..9348882e6cf 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs @@ -20,8 +20,8 @@ public RuntimeInfoProvider(IModuleManager moduleManager) public Task> GetModules(CancellationToken token) => this.moduleManager.GetModules(token); - public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) => - this.moduleManager.GetModuleLogs(module, follow, tail, since, until, cancellationToken); + public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) => + this.moduleManager.GetModuleLogs(module, follow, tail, since, until, includeTimestamp, cancellationToken); public Task GetSystemInfo(CancellationToken token) => this.moduleManager.GetSystemInfoAsync(token); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs index 19c9c634d35..17003318b4f 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs @@ -25,6 +25,7 @@ abstract class ModuleManagementHttpClientVersioned const string LogsUrlTailParameter = "tail"; const string LogsUrlSinceParameter = "since"; const string LogsUrlUntilParameter = "until"; + const string LogsIncludeTimestampParameter = "timestamps"; static readonly TimeSpan DefaultOperationTimeout = TimeSpan.FromMinutes(5); @@ -84,7 +85,7 @@ protected ModuleManagementHttpClientVersioned( public abstract Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token); - public virtual async Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) + public virtual async Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) { using (HttpClient httpClient = HttpClientHelper.GetHttpClient(this.ManagementUri)) { @@ -93,6 +94,7 @@ public virtual async Task GetModuleLogs(string module, bool follow, Opti logsUrl.AppendFormat(CultureInfo.InvariantCulture, LogsUrlTemplate, baseUrl, module, this.Version.Name, follow.ToString().ToLowerInvariant()); since.ForEach(s => logsUrl.AppendFormat($"&{LogsUrlSinceParameter}={Uri.EscapeUriString(s)}")); until.ForEach(u => logsUrl.AppendFormat($"&{LogsUrlUntilParameter}={Uri.EscapeUriString(u)}")); + includeTimestamp.ForEach(b => logsUrl.AppendFormat($"&{LogsIncludeTimestampParameter}={b.ToString().ToLower()}")); if (!(tail.HasValue && since.HasValue && until.HasValue)) { @@ -100,6 +102,7 @@ public virtual async Task GetModuleLogs(string module, bool follow, Opti } var logsUri = new Uri(logsUrl.ToString()); + var httpRequest = new HttpRequestMessage(HttpMethod.Get, logsUri); Stream stream = await this.Execute( async () => @@ -108,6 +111,7 @@ public virtual async Task GetModuleLogs(string module, bool follow, Opti return await httpResponseMessage.Content.ReadAsStreamAsync(); }, $"Get logs for {module}"); + return stream; } } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs index f57f218c92f..ed9e34044ec 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs @@ -49,6 +49,7 @@ public async Task UploadLogs(string uri, string id, byte[] payload, LogsContentE string blobName = this.GetBlobName(id, GetLogsExtension(logsContentEncoding, logsContentType)); var container = new CloudBlobContainer(containerUri); Events.Uploading(blobName, container.Name); + await ExecuteWithRetry( () => { diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs index a7171d6338f..b44b354e272 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs @@ -76,7 +76,7 @@ public async Task> GetModules(CancellationToken c return await Task.FromResult(moduleRuntimeInfoList); } - public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) + public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) { return Task.FromResult(Stream.Null); } diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs index 48c2b028e07..ee78c0e7795 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs @@ -127,7 +127,7 @@ public async Task GetTextWithRegexFilterTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(TestLogBytes); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); // Act IEnumerable textLines = await logsProcessor.GetText(moduleId, stream, filter); @@ -149,7 +149,7 @@ public async Task GetMessagesWithRegexFilterTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(TestLogBytes); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); // Act IEnumerable logMessages = await logsProcessor.GetMessages(moduleId, stream, filter); @@ -186,7 +186,7 @@ public async Task GetTextWithLogLevelFilterTest(int logLevel, List expected var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None()); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.None()); // Act List textLines = (await logsProcessor.GetText(moduleId, stream, filter)).ToList(); @@ -211,7 +211,7 @@ public async Task GetMessagesWithLogLevelFilterTest(int logLevel, List expe var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None()); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.None()); // Act IEnumerable logMessages = await logsProcessor.GetMessages(moduleId, stream, filter); @@ -239,7 +239,7 @@ public async Task GetTextWithMultipleFiltersTest(int logLevel, string regex, Lis var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); // Act List textLines = (await logsProcessor.GetText(moduleId, stream, filter)).ToList(); @@ -264,7 +264,7 @@ public async Task GetMessagesWithMultipleFiltersTest(int logLevel, string regex, var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); // Act IEnumerable logMessages = await logsProcessor.GetMessages(moduleId, stream, filter); @@ -293,7 +293,7 @@ public async Task ProcessStreamTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), false); var receivedBytes = new List(); @@ -328,7 +328,7 @@ public async Task ProcessStreamToMessageTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, filter, LogOutputFraming.None, Option.None(), false); var receivedBytes = new List(); @@ -365,7 +365,7 @@ public async Task ProcessStreamWithGzipTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), false); var receivedBytes = new List(); @@ -400,7 +400,7 @@ public async Task ProcessStreamWithSimpleFramingTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, filter, LogOutputFraming.SimpleLength, Option.None(), false); var receivedBytes = new List(); @@ -435,7 +435,7 @@ public async Task ProcessStreamToMessageWithSimpleFramingTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, filter, LogOutputFraming.SimpleLength, Option.None(), false); var receivedBytes = new List(); @@ -473,7 +473,7 @@ public async Task ProcessStreamWithGroupingGzipTest() string moduleId = "mod1"; var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); var stream1 = new MemoryStream(DockerFraming.Frame(TestLogTexts)); var logOptions1 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), false); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs index c0ef444488f..8f25f91b4b6 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs @@ -30,17 +30,18 @@ public class LogsProviderTest public static IEnumerable GetNeedToProcessStreamTestData() { yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false), false }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), false }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), false }; yield return new object[] { new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.None()), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.SimpleLength, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.Some(new LogsOutputGroupingConfig(100, TimeSpan.FromMilliseconds(1000))), false), false }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.None(), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.SimpleLength, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.Some(new LogsOutputGroupingConfig(100, TimeSpan.FromMilliseconds(1000))), false), false }; } [Fact] + [Trait("Category", "GetLogsTests")] public async Task GetLogsAsTextTest() { // Arrange @@ -49,11 +50,13 @@ public async Task GetLogsAsTextTest() string moduleId = "mod1"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.None(); CancellationToken cancellationToken = CancellationToken.None; string expectedLogText = TestLogTexts.Join(string.Empty); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); @@ -70,19 +73,22 @@ public async Task GetLogsAsTextTest() } [Fact] + [Trait("Category", "GetLogsTests")] public async Task GetLogsAsTextWithCompressionTest() { // Arrange string iotHub = "foo.azure-devices.net"; - string deviceId = "dev1"; - string moduleId = "mod1"; + string deviceId = "dev2"; + string moduleId = "mod2"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.None(); CancellationToken cancellationToken = CancellationToken.None; string expectedLogText = TestLogTexts.Join(string.Empty); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); @@ -108,16 +114,20 @@ public async Task GetLogsAsJsonTest() string moduleId = "mod1"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.Some(true); CancellationToken cancellationToken = CancellationToken.None; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + // Note: EdgeAgent automatically includes the timestamp for log parsing by default for content type JSON + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor); - var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); + ModuleLogFilter filter = new ModuleLogFilter(tail, since, until, Option.None(), includeTimestamp, Option.None()); + var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, filter, LogOutputFraming.None, Option.None(), false); // Act byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken); @@ -149,16 +159,20 @@ public async Task GetLogsAsJsonWithCompressionTest() string moduleId = "mod1"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.Some(true); CancellationToken cancellationToken = CancellationToken.None; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + // Note: EdgeAgent automatically includes the timestamp for log parsing by default for content type JSON + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor); - var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); + ModuleLogFilter filter = new ModuleLogFilter(tail, since, until, Option.None(), includeTimestamp, Option.None()); + var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, filter, LogOutputFraming.None, Option.None(), false); // Act byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken); @@ -183,19 +197,22 @@ public async Task GetLogsAsJsonWithCompressionTest() } [Fact] + [Trait("Category", "GetLogsTests")] public async Task GetLogsStreamTest() { // Arrange string iotHub = "foo.azure-devices.net"; - string deviceId = "dev1"; - string moduleId = "mod1"; + string deviceId = "dev3"; + string moduleId = "mod3"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.None(); CancellationToken cancellationToken = CancellationToken.None; byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); @@ -234,7 +251,7 @@ public async Task GetLogsStreamWithFiltersTest() byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); @@ -242,7 +259,7 @@ public async Task GetLogsStreamWithFiltersTest() var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor); - var filter = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.Some("Starting")); + var filter = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.None(), Option.Some("Starting")); var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), true); var receivedBytes = new List(); @@ -278,8 +295,8 @@ public async Task GetLogsStreamWithMultipleModulesTest() string moduleId1 = "mod1"; string moduleId2 = "mod2"; - var filter1 = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.Some("Starting")); - var filter2 = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.Some("bad")); + var filter1 = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.None(), Option.Some("Starting")); + var filter2 = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.Some("bad")); byte[] dockerLogsStreamBytes1 = DockerFraming.Frame(TestLogTexts); byte[] dockerLogsStreamBytes2 = DockerFraming.Frame(TestLogTexts); @@ -291,9 +308,9 @@ public async Task GetLogsStreamWithMultipleModulesTest() }; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes1)); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, Option.None(), Option.None(), Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, Option.None(), Option.None(), Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes2)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(modulesList); @@ -351,7 +368,7 @@ public async Task GetLogsStreamWithMultipleModulesWithRegexMatchTest() string moduleId1 = "mod1"; string moduleId2 = "mod2"; - var filter = new ModuleLogFilter(tail, since, Option.None(), Option.None(), Option.Some("bad")); + var filter = new ModuleLogFilter(tail, since, Option.None(), Option.None(), Option.None(), Option.Some("bad")); byte[] dockerLogsStreamBytes1 = DockerFraming.Frame(TestLogTexts); byte[] dockerLogsStreamBytes2 = DockerFraming.Frame(TestLogTexts); @@ -363,9 +380,9 @@ public async Task GetLogsStreamWithMultipleModulesWithRegexMatchTest() }; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes1)); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes2)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(modulesList); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs index bae81cabad8..75f76e90f7e 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs @@ -46,7 +46,7 @@ public static IEnumerable GetIdsToProcessTestData() { var logOptions1 = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); var logOptions2 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); - var logOptions3 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false); + var logOptions3 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false); yield return new object[] { diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs index 2d6d82a0063..879f12a35a5 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs @@ -19,7 +19,7 @@ public class LogsRequestHandlerTests [Fact] public async Task GetJsonLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Json; @@ -84,7 +84,7 @@ public async Task GetJsonLogsTest() [Fact] public async Task GetTextLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Text; @@ -147,7 +147,7 @@ public async Task GetTextLogsTest() [Fact] public async Task GetJsonGzipLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.Gzip; LogsContentType contentType = LogsContentType.Json; @@ -212,7 +212,7 @@ public async Task GetJsonGzipLogsTest() [Fact] public async Task GetTextGzipLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.Gzip; LogsContentType contentType = LogsContentType.Text; diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs index b34f91b32a3..da32aaa92ea 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs @@ -31,15 +31,15 @@ public static IEnumerable GetLogsUploadRequestHandlerData() yield return new object[] { @"{""schemaVersion"":""1.0"",""sasUrl"":"""",""items"":{""id"":""edgeHub""},""encoding"":""none"",""contentType"":""json""}".Replace("", sasUrl), "edgeHub", sasUrl, LogsContentEncoding.None, LogsContentType.Json, ModuleLogFilter.Empty }; sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some(true), Option.Some("ERR")); yield return new object[] { @"{""schemaVersion"":""1.0"",""sasUrl"":"""",""items"":{""id"":""edgeAgent"",""filter"":}}".Replace("", sasUrl).Replace("", filter.ToJson()), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json, filter }; sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - filter = new ModuleLogFilter(Option.None(), Option.Some("1501000"), Option.None(), Option.None(), Option.Some("ERR")); + filter = new ModuleLogFilter(Option.None(), Option.Some("1501000"), Option.None(), Option.None(), Option.Some(true), Option.Some("ERR")); yield return new object[] { @"{""schemaVersion"":""1.5"",""sasUrl"":"""",""items"":{""id"":""edgeAgent"",""filter"":}}".Replace("", sasUrl).Replace("", filter.ToJson()), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json, filter }; sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - filter = new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.Some(3), Option.None()); + filter = new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.Some(3), Option.Some(true), Option.None()); yield return new object[] { @"{""schemaVersion"":""1.2"",""sasUrl"":"""",""items"":{""id"":""edgeAgent"",""filter"":}}".Replace("", sasUrl).Replace("", filter.ToJson()), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json, filter }; } @@ -63,7 +63,7 @@ public async Task TestLogsUploadRequest(string payload, string id, string sasUrl } else { - logsProvider.Setup(l => l.GetLogs(id, moduleLogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(id, It.IsAny(), It.IsAny())) .ReturnsAsync(uploadBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, id, uploadBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); @@ -73,6 +73,7 @@ public async Task TestLogsUploadRequest(string payload, string id, string sasUrl { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, string.Empty, 0, Option.None(), Option.None()) }; + var runtimeInfoProvider = Mock.Of(r => r.GetModules(It.IsAny()) == Task.FromResult(moduleRuntimeInfoList)); // Act @@ -98,7 +99,8 @@ public async Task TestLogsUploadAllTaskRequest() { // Arrange string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + // Note: When LogsContentType = Json, edgeAgent automatically set the "includeTimestamp" to be true. + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some(true), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Json; @@ -134,21 +136,21 @@ public async Task TestLogsUploadAllTaskRequest() var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None(), false); var mod1LogBytes = new byte[100]; - logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(mod1, It.IsAny(), It.IsAny())) .ReturnsAsync(mod1LogBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, mod1, mod1LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module2LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None(), false); var mod2LogBytes = new byte[80]; - logsProvider.Setup(l => l.GetLogs(mod2, module2LogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(mod2, It.IsAny(), It.IsAny())) .ReturnsAsync(mod2LogBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, mod2, mod2LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module3LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None(), false); var mod3LogBytes = new byte[120]; - logsProvider.Setup(l => l.GetLogs(mod3, module3LogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(mod3, It.IsAny(), It.IsAny())) .ReturnsAsync(mod3LogBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, mod3, mod3LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs index 73511aa7226..2244041a3f4 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs @@ -285,7 +285,7 @@ public async Task GetModuleLogsTest() var runtimeInfoProvider = await RuntimeInfoProvider.CreateAsync(dockerClient); // Act - Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), CancellationToken.None); + Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), Option.None(), CancellationToken.None); // Assert Assert.NotNull(receivedContainerLogsParameters); @@ -301,7 +301,7 @@ public async Task GetModuleLogsTest() Assert.Equal(dummyLogs, receivedLogs); // Act - receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.Some(1000), Option.None(), Option.None(), CancellationToken.None); + receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.Some(1000), Option.None(), Option.None(), Option.None(), CancellationToken.None); // Assert Assert.NotNull(receivedContainerLogsParameters); @@ -317,7 +317,7 @@ public async Task GetModuleLogsTest() Assert.Equal(dummyLogs, receivedLogs); // Act - receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.None(), Option.Some("1552887267"), Option.None(), CancellationToken.None); + receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.None(), Option.Some("1552887267"), Option.None(), Option.None(), CancellationToken.None); // Assert Assert.NotNull(receivedContainerLogsParameters); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs index 617b97313b8..647bb124857 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs @@ -262,7 +262,7 @@ public async Task ModuleLogsTest(string serverApiVersion, string clientApiVersio IModuleManager client = new ModuleManagementHttpClient(this.serverUrl, serverApiVersion, clientApiVersion); // Act - Stream logsStream = await client.GetModuleLogs("edgeHub", false, Option.None(), Option.None(), Option.None(), CancellationToken.None); + Stream logsStream = await client.GetModuleLogs("edgeHub", false, Option.None(), Option.None(), Option.None(), Option.Some(false), CancellationToken.None); // Assert Assert.NotNull(logsStream); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs index c211cecdcf6..0498a172e60 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs @@ -101,13 +101,13 @@ public async Task GetLogsTest() Stream GetLogsStream() => new MemoryStream(Encoding.UTF8.GetBytes(dummyLogs)); var moduleManager = new Mock(); - moduleManager.Setup(m => m.GetModuleLogs(id, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny>(), It.IsAny())) + moduleManager.Setup(m => m.GetModuleLogs(id, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny>(), Option.None(), It.IsAny())) .ReturnsAsync(GetLogsStream); var runtimeInfoProvider = new RuntimeInfoProvider(moduleManager.Object); // Act - Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), CancellationToken.None); + Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), Option.None(), CancellationToken.None); // Assert var buffer = new byte[1024]; diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs index 50e5c779d8c..c8108040ca8 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs @@ -39,7 +39,7 @@ public async Task HandleTest() string id = "m1"; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), It.IsAny())) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), Option.None(), It.IsAny())) .ReturnsAsync(new MemoryStream(buffer)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); @@ -86,7 +86,7 @@ public async Task HandleWithCancellationTest() string id = "m1"; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), It.IsAny())) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), Option.None(), It.IsAny())) .ReturnsAsync(new MemoryStream(buffer)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs index ca135568966..f934805b955 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs @@ -36,7 +36,7 @@ public class DummyModuleManager : IModuleManager public Task PrepareUpdateAsync(ModuleSpec moduleSpec) => throw new NotImplementedException(); - public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token) => throw new NotImplementedException(); } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs b/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs index 8362cf2168a..db0a1618d9c 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs @@ -59,7 +59,7 @@ await this.runtime.DeployConfigurationAsync( string since = DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd'T'HH:mm:sZ"); string until = DateTime.Now.AddDays(+1).ToString("yyyy-MM-dd'T'HH:mm:sZ"); - var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.Some(10), Option.Some(since), Option.Some(until), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); + var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.Some(10), Option.Some(since), Option.Some(until), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); var result = await this.IotHub.InvokeMethodAsync(this.runtime.DeviceId, ConfigModuleName.EdgeAgent, new CloudToDeviceMethod("GetModuleLogs", TimeSpan.FromSeconds(300), TimeSpan.FromSeconds(300)).SetPayloadJson(JsonConvert.SerializeObject(request)), token); @@ -88,7 +88,7 @@ await this.runtime.DeployConfigurationAsync( Context.Current.NestedEdge); await Task.Delay(30000); - var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); + var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); var result = await this.IotHub.InvokeMethodAsync(this.runtime.DeviceId, ConfigModuleName.EdgeAgent, new CloudToDeviceMethod("GetModuleLogs", TimeSpan.FromSeconds(300), TimeSpan.FromSeconds(300)).SetPayloadJson(JsonConvert.SerializeObject(request)), token); @@ -126,7 +126,7 @@ await this.runtime.DeployConfigurationAsync( await Task.Delay(10000); // check it restarted - var logsRequest = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); + var logsRequest = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); result = await this.IotHub.InvokeMethodAsync(this.runtime.DeviceId, ConfigModuleName.EdgeAgent, new CloudToDeviceMethod("GetModuleLogs", TimeSpan.FromSeconds(300), TimeSpan.FromSeconds(300)).SetPayloadJson(JsonConvert.SerializeObject(logsRequest)), token); Assert.AreEqual((int)HttpStatusCode.OK, result.Status);