diff --git a/builds/checkin/dotnet.yaml b/builds/checkin/dotnet.yaml index 0127bdd9802..ce38f7fc2e2 100644 --- a/builds/checkin/dotnet.yaml +++ b/builds/checkin/dotnet.yaml @@ -45,7 +45,12 @@ jobs: displayName: Test inputs: filePath: scripts/linux/runTests.sh - arguments: '"--filter Category=Unit"' + arguments: '"--filter Category=Unit&Category!=GetLogsTests"' + - task: Bash@3 + displayName: Run GetLogsTests Tests + inputs: + filePath: scripts/linux/runTests.sh + arguments: '"--filter Category=GetLogsTests"' - task: PublishTestResults@2 displayName: Publish Test Results inputs: @@ -75,7 +80,19 @@ jobs: testSelector: testAssemblies testAssemblyVer2: | target\*Test.dll - testFiltercriteria: Category=Unit + testFiltercriteria: Category=Unit&Category!=GetLogsTests + runInParallel: true + runTestsInIsolation: true + codeCoverageEnabled: true + runSettingsFile: CodeCoverage.runsettings + publishRunAttachments: true + - task: VSTest@2 + displayName: Run GetLogsTests unit tests with code coverage + inputs: + testSelector: testAssemblies + testAssemblyVer2: | + target\*Test.dll + testFiltercriteria: Category=GetLogsTests runInParallel: true runTestsInIsolation: true codeCoverageEnabled: true diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs index c2288987186..455d707d7c5 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IRuntimeInfoProvider.cs @@ -16,7 +16,7 @@ public interface IRuntimeInfoProvider { Task> GetModules(CancellationToken cancellationToken); - Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken); + Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken); Task GetSystemInfo(CancellationToken cancellationToken); } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs index 5d32b21300e..df7481bc5ea 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogMessageParser.cs @@ -3,6 +3,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs { extern alias akka; using akka::Akka.IO; + using Microsoft.Azure.Devices.Edge.Util; public interface ILogMessageParser { diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs index 0e55cd40350..2a85b84befd 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogMessageParser.cs @@ -29,7 +29,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs public class LogMessageParser : ILogMessageParser { const int DefaultLogLevel = 6; - const string LogRegexPattern = @"^(<(?\d)>)?\s*((?\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d{3}\s[+-]\d{2}:\d{2})\s)?\s*(?.*)"; + const string LogRegexPattern = @"^((?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{9}Z))?\s*(<(?\d)>)?\s*((?\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d{3}\s[+-]\d{2}:\d{2})\s)?\s*(?.*)"; readonly string iotHubName; readonly string deviceId; @@ -48,6 +48,7 @@ internal static ModuleLogMessageData GetLogMessage(ByteString arg, string iotHub string stream = GetStream(arg[0]); ByteString payload = arg.Slice(8); string payloadString = payload.ToString(Encoding.UTF8); + (int logLevel, Option timeStamp, string logText) = ParseLogText(payloadString); var moduleLogMessage = new ModuleLogMessageData(iotHubName, deviceId, moduleId, stream, logLevel, timeStamp, logText, arg, payloadString); return moduleLogMessage; @@ -58,14 +59,24 @@ internal static ModuleLogMessageData GetLogMessage(ByteString arg, string iotHub internal static (int logLevel, Option timeStamp, string text) ParseLogText(string value) { var regex = new Regex(LogRegexPattern); + var match = regex.Match(value); int logLevel = DefaultLogLevel; string text = value; Option timeStamp = Option.None(); if (match.Success) { + var dtsg = match.Groups["dockerTimestamp"]; + if (dtsg?.Length > 0) + { + if (DateTime.TryParse(dtsg.Value, out DateTime dt)) + { + timeStamp = Option.Some(dt); + } + } + var tsg = match.Groups["timestamp"]; - if (tsg?.Length > 0) + if (tsg?.Length > 0 && !timeStamp.HasValue) { if (DateTime.TryParse(tsg.Value, out DateTime dt)) { diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs index 7678066c630..1533dcfd744 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs @@ -26,7 +26,12 @@ public async Task GetLogs(string id, ModuleLogOptions logOptions, Cancel { Preconditions.CheckNotNull(logOptions, nameof(logOptions)); - Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, false, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, cancellationToken); + if (logOptions.ContentType == LogsContentType.Json) + { + logOptions.Filter.IncludeTimestamp = Option.Some(true); + } + + Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, false, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, logOptions.Filter.IncludeTimestamp, cancellationToken); Events.ReceivedStream(id); byte[] logBytes = await this.GetProcessedLogs(id, logsStream, logOptions); @@ -60,7 +65,7 @@ internal async Task GetLogsStreamInternal(string id, ModuleLogOptions logOptions Preconditions.CheckNotNull(logOptions, nameof(logOptions)); Preconditions.CheckNotNull(callback, nameof(callback)); - Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, logOptions.Follow, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, cancellationToken); + Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, logOptions.Follow, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, logOptions.Filter.IncludeTimestamp, cancellationToken); Events.ReceivedStream(id); await this.logsProcessor.ProcessLogsStream(id, logsStream, logOptions, callback); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs index b88ddd3414a..ddbe9248b56 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ModuleLogFilter.cs @@ -10,21 +10,22 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs public class ModuleLogFilter : IEquatable { - public static ModuleLogFilter Empty = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); + public static ModuleLogFilter Empty = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); - public ModuleLogFilter(Option tail, Option since, Option until, Option logLevel, Option regex) + public ModuleLogFilter(Option tail, Option since, Option until, Option logLevel, Option includeTimestamp, Option regex) { this.Tail = tail; this.Since = since; this.Until = until; this.LogLevel = logLevel; this.RegexString = regex; + this.IncludeTimestamp = includeTimestamp; this.Regex = regex.Map(r => new Regex(r)); } [JsonConstructor] - ModuleLogFilter(int? tail, string since, string until, int? loglevel, string regex) - : this(Option.Maybe(tail), Option.Maybe(since), Option.Maybe(until), Option.Maybe(loglevel), Option.Maybe(regex)) + ModuleLogFilter(int? tail, string since, string until, int? loglevel, bool? includeTimestamp, string regex) + : this(Option.Maybe(tail), Option.Maybe(since), Option.Maybe(until), Option.Maybe(loglevel), Option.Maybe(includeTimestamp), Option.Maybe(regex)) { } @@ -44,6 +45,9 @@ public ModuleLogFilter(Option tail, Option since, Option un [JsonConverter(typeof(OptionConverter), true)] public Option LogLevel { get; } + [JsonIgnore] + public Option IncludeTimestamp { get; set; } + [JsonIgnore] public Option Regex { get; } @@ -59,6 +63,7 @@ public bool Equals(ModuleLogFilter other) this.Tail.Equals(other.Tail) && this.Since.Equals(other.Since) && this.LogLevel.Equals(other.LogLevel) && + this.IncludeTimestamp.Equals(other.IncludeTimestamp) && this.RegexString.Equals(other.RegexString); public override int GetHashCode() @@ -67,6 +72,7 @@ public override int GetHashCode() hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.Tail); hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.Since); hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.LogLevel); + hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.IncludeTimestamp); hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(this.RegexString); return hashCode; } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs index 4bdaf1e6274..d360fd1f7b8 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Docker/RuntimeInfoProvider.cs @@ -79,7 +79,7 @@ await Task.WhenAll( return modules; } - public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) + public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) { var containerLogsParameters = new ContainerLogsParameters { @@ -89,6 +89,7 @@ public Task GetModuleLogs(string module, bool follow, Option tail, }; tail.ForEach(t => containerLogsParameters.Tail = t.ToString()); since.ForEach(t => containerLogsParameters.Since = t.ToString()); + includeTimestamp.ForEach(t => containerLogsParameters.Timestamps = t); return this.client.Containers.GetContainerLogsAsync(module, containerLogsParameters, cancellationToken); } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs index 4c25467fccc..bc34d786c3f 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs @@ -33,7 +33,7 @@ public interface IModuleManager Task PrepareUpdateAsync(ModuleSpec moduleSpec); - Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken); + Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken); Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token); } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs index 58b4b59e4b9..9815d07d728 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/ModuleManagementHttpClient.cs @@ -57,8 +57,8 @@ public ModuleManagementHttpClient(Uri managementUri, string serverSupportedApiVe public Task ReprovisionDeviceAsync() => this.inner.ReprovisionDeviceAsync(); - public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) => - this.inner.GetModuleLogs(name, follow, tail, since, until, cancellationToken); + public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) => + this.inner.GetModuleLogs(name, follow, tail, since, until, includeTimestamp, cancellationToken); public Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token) => this.inner.GetSupportBundle(since, until, iothubHostname, edgeRuntimeOnly, token); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs index fded4713e33..9348882e6cf 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/RuntimeInfoProvider.cs @@ -20,8 +20,8 @@ public RuntimeInfoProvider(IModuleManager moduleManager) public Task> GetModules(CancellationToken token) => this.moduleManager.GetModules(token); - public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) => - this.moduleManager.GetModuleLogs(module, follow, tail, since, until, cancellationToken); + public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) => + this.moduleManager.GetModuleLogs(module, follow, tail, since, until, includeTimestamp, cancellationToken); public Task GetSystemInfo(CancellationToken token) => this.moduleManager.GetSystemInfoAsync(token); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs index 19c9c634d35..17003318b4f 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/versioning/ModuleManagementHttpClientVersioned.cs @@ -25,6 +25,7 @@ abstract class ModuleManagementHttpClientVersioned const string LogsUrlTailParameter = "tail"; const string LogsUrlSinceParameter = "since"; const string LogsUrlUntilParameter = "until"; + const string LogsIncludeTimestampParameter = "timestamps"; static readonly TimeSpan DefaultOperationTimeout = TimeSpan.FromMinutes(5); @@ -84,7 +85,7 @@ protected ModuleManagementHttpClientVersioned( public abstract Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token); - public virtual async Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) + public virtual async Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) { using (HttpClient httpClient = HttpClientHelper.GetHttpClient(this.ManagementUri)) { @@ -93,6 +94,7 @@ public virtual async Task GetModuleLogs(string module, bool follow, Opti logsUrl.AppendFormat(CultureInfo.InvariantCulture, LogsUrlTemplate, baseUrl, module, this.Version.Name, follow.ToString().ToLowerInvariant()); since.ForEach(s => logsUrl.AppendFormat($"&{LogsUrlSinceParameter}={Uri.EscapeUriString(s)}")); until.ForEach(u => logsUrl.AppendFormat($"&{LogsUrlUntilParameter}={Uri.EscapeUriString(u)}")); + includeTimestamp.ForEach(b => logsUrl.AppendFormat($"&{LogsIncludeTimestampParameter}={b.ToString().ToLower()}")); if (!(tail.HasValue && since.HasValue && until.HasValue)) { @@ -100,6 +102,7 @@ public virtual async Task GetModuleLogs(string module, bool follow, Opti } var logsUri = new Uri(logsUrl.ToString()); + var httpRequest = new HttpRequestMessage(HttpMethod.Get, logsUri); Stream stream = await this.Execute( async () => @@ -108,6 +111,7 @@ public virtual async Task GetModuleLogs(string module, bool follow, Opti return await httpResponseMessage.Content.ReadAsStreamAsync(); }, $"Get logs for {module}"); + return stream; } } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs index f57f218c92f..ed9e34044ec 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobRequestsUploader.cs @@ -49,6 +49,7 @@ public async Task UploadLogs(string uri, string id, byte[] payload, LogsContentE string blobName = this.GetBlobName(id, GetLogsExtension(logsContentEncoding, logsContentType)); var container = new CloudBlobContainer(containerUri); Events.Uploading(blobName, container.Name); + await ExecuteWithRetry( () => { diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs index a7171d6338f..b44b354e272 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesRuntimeInfoProvider.cs @@ -76,7 +76,7 @@ public async Task> GetModules(CancellationToken c return await Task.FromResult(moduleRuntimeInfoList); } - public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) + public Task GetModuleLogs(string module, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) { return Task.FromResult(Stream.Null); } diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs index 48c2b028e07..ee78c0e7795 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProcessorTest.cs @@ -127,7 +127,7 @@ public async Task GetTextWithRegexFilterTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(TestLogBytes); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); // Act IEnumerable textLines = await logsProcessor.GetText(moduleId, stream, filter); @@ -149,7 +149,7 @@ public async Task GetMessagesWithRegexFilterTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(TestLogBytes); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.Some(regex)); // Act IEnumerable logMessages = await logsProcessor.GetMessages(moduleId, stream, filter); @@ -186,7 +186,7 @@ public async Task GetTextWithLogLevelFilterTest(int logLevel, List expected var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None()); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.None()); // Act List textLines = (await logsProcessor.GetText(moduleId, stream, filter)).ToList(); @@ -211,7 +211,7 @@ public async Task GetMessagesWithLogLevelFilterTest(int logLevel, List expe var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None()); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.None()); // Act IEnumerable logMessages = await logsProcessor.GetMessages(moduleId, stream, filter); @@ -239,7 +239,7 @@ public async Task GetTextWithMultipleFiltersTest(int logLevel, string regex, Lis var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); // Act List textLines = (await logsProcessor.GetText(moduleId, stream, filter)).ToList(); @@ -264,7 +264,7 @@ public async Task GetMessagesWithMultipleFiltersTest(int logLevel, string regex, var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); // Act IEnumerable logMessages = await logsProcessor.GetMessages(moduleId, stream, filter); @@ -293,7 +293,7 @@ public async Task ProcessStreamTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), false); var receivedBytes = new List(); @@ -328,7 +328,7 @@ public async Task ProcessStreamToMessageTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, filter, LogOutputFraming.None, Option.None(), false); var receivedBytes = new List(); @@ -365,7 +365,7 @@ public async Task ProcessStreamWithGzipTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), false); var receivedBytes = new List(); @@ -400,7 +400,7 @@ public async Task ProcessStreamWithSimpleFramingTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, filter, LogOutputFraming.SimpleLength, Option.None(), false); var receivedBytes = new List(); @@ -435,7 +435,7 @@ public async Task ProcessStreamToMessageWithSimpleFramingTest() var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); var stream = new MemoryStream(DockerFraming.Frame(TestLogTexts)); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.Some(regex)); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.Some(logLevel), Option.None(), Option.Some(regex)); var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, filter, LogOutputFraming.SimpleLength, Option.None(), false); var receivedBytes = new List(); @@ -473,7 +473,7 @@ public async Task ProcessStreamWithGroupingGzipTest() string moduleId = "mod1"; var logMessageParser = new LogMessageParser(iotHub, deviceId); var logsProcessor = new LogsProcessor(logMessageParser); - var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); + var filter = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None()); var stream1 = new MemoryStream(DockerFraming.Frame(TestLogTexts)); var logOptions1 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), false); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs index c0ef444488f..8f25f91b4b6 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsProviderTest.cs @@ -30,17 +30,18 @@ public class LogsProviderTest public static IEnumerable GetNeedToProcessStreamTestData() { yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false), false }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), false }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), false }; yield return new object[] { new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.None()), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.SimpleLength, Option.None(), false), true }; - yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.Some(new LogsOutputGroupingConfig(100, TimeSpan.FromMilliseconds(1000))), false), false }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.None(), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.Some(3), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.Some("foo")), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.SimpleLength, Option.None(), false), true }; + yield return new object[] { new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(10), Option.Some("100"), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.Some(new LogsOutputGroupingConfig(100, TimeSpan.FromMilliseconds(1000))), false), false }; } [Fact] + [Trait("Category", "GetLogsTests")] public async Task GetLogsAsTextTest() { // Arrange @@ -49,11 +50,13 @@ public async Task GetLogsAsTextTest() string moduleId = "mod1"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.None(); CancellationToken cancellationToken = CancellationToken.None; string expectedLogText = TestLogTexts.Join(string.Empty); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); @@ -70,19 +73,22 @@ public async Task GetLogsAsTextTest() } [Fact] + [Trait("Category", "GetLogsTests")] public async Task GetLogsAsTextWithCompressionTest() { // Arrange string iotHub = "foo.azure-devices.net"; - string deviceId = "dev1"; - string moduleId = "mod1"; + string deviceId = "dev2"; + string moduleId = "mod2"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.None(); CancellationToken cancellationToken = CancellationToken.None; string expectedLogText = TestLogTexts.Join(string.Empty); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); @@ -108,16 +114,20 @@ public async Task GetLogsAsJsonTest() string moduleId = "mod1"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.Some(true); CancellationToken cancellationToken = CancellationToken.None; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + // Note: EdgeAgent automatically includes the timestamp for log parsing by default for content type JSON + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor); - var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); + ModuleLogFilter filter = new ModuleLogFilter(tail, since, until, Option.None(), includeTimestamp, Option.None()); + var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, filter, LogOutputFraming.None, Option.None(), false); // Act byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken); @@ -149,16 +159,20 @@ public async Task GetLogsAsJsonWithCompressionTest() string moduleId = "mod1"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.Some(true); CancellationToken cancellationToken = CancellationToken.None; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, Option.None(), cancellationToken)) + // Note: EdgeAgent automatically includes the timestamp for log parsing by default for content type JSON + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts))); var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor); - var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); + ModuleLogFilter filter = new ModuleLogFilter(tail, since, until, Option.None(), includeTimestamp, Option.None()); + var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, filter, LogOutputFraming.None, Option.None(), false); // Act byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken); @@ -183,19 +197,22 @@ public async Task GetLogsAsJsonWithCompressionTest() } [Fact] + [Trait("Category", "GetLogsTests")] public async Task GetLogsStreamTest() { // Arrange string iotHub = "foo.azure-devices.net"; - string deviceId = "dev1"; - string moduleId = "mod1"; + string deviceId = "dev3"; + string moduleId = "mod3"; Option tail = Option.None(); Option since = Option.None(); + Option until = Option.None(); + Option includeTimestamp = Option.None(); CancellationToken cancellationToken = CancellationToken.None; byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, until, includeTimestamp, cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); @@ -234,7 +251,7 @@ public async Task GetLogsStreamWithFiltersTest() byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts); var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); @@ -242,7 +259,7 @@ public async Task GetLogsStreamWithFiltersTest() var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId)); var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor); - var filter = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.Some("Starting")); + var filter = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.None(), Option.Some("Starting")); var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter, LogOutputFraming.None, Option.None(), true); var receivedBytes = new List(); @@ -278,8 +295,8 @@ public async Task GetLogsStreamWithMultipleModulesTest() string moduleId1 = "mod1"; string moduleId2 = "mod2"; - var filter1 = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.Some("Starting")); - var filter2 = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.Some("bad")); + var filter1 = new ModuleLogFilter(tail, since, Option.None(), Option.Some(6), Option.None(), Option.Some("Starting")); + var filter2 = new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.Some("bad")); byte[] dockerLogsStreamBytes1 = DockerFraming.Frame(TestLogTexts); byte[] dockerLogsStreamBytes2 = DockerFraming.Frame(TestLogTexts); @@ -291,9 +308,9 @@ public async Task GetLogsStreamWithMultipleModulesTest() }; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes1)); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, Option.None(), Option.None(), Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, Option.None(), Option.None(), Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes2)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(modulesList); @@ -351,7 +368,7 @@ public async Task GetLogsStreamWithMultipleModulesWithRegexMatchTest() string moduleId1 = "mod1"; string moduleId2 = "mod2"; - var filter = new ModuleLogFilter(tail, since, Option.None(), Option.None(), Option.Some("bad")); + var filter = new ModuleLogFilter(tail, since, Option.None(), Option.None(), Option.None(), Option.Some("bad")); byte[] dockerLogsStreamBytes1 = DockerFraming.Frame(TestLogTexts); byte[] dockerLogsStreamBytes2 = DockerFraming.Frame(TestLogTexts); @@ -363,9 +380,9 @@ public async Task GetLogsStreamWithMultipleModulesWithRegexMatchTest() }; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes1)); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, tail, since, Option.None(), cancellationToken)) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, tail, since, Option.None(), Option.None(), cancellationToken)) .ReturnsAsync(new MemoryStream(dockerLogsStreamBytes2)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(modulesList); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs index bae81cabad8..75f76e90f7e 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/logs/LogsRequestToOptionsMapperTests.cs @@ -46,7 +46,7 @@ public static IEnumerable GetIdsToProcessTestData() { var logOptions1 = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); var logOptions2 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None(), false); - var logOptions3 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false); + var logOptions3 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.None(), Option.None(), Option.None()), LogOutputFraming.None, Option.None(), false); yield return new object[] { diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs index 2d6d82a0063..879f12a35a5 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsRequestHandlerTests.cs @@ -19,7 +19,7 @@ public class LogsRequestHandlerTests [Fact] public async Task GetJsonLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Json; @@ -84,7 +84,7 @@ public async Task GetJsonLogsTest() [Fact] public async Task GetTextLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Text; @@ -147,7 +147,7 @@ public async Task GetTextLogsTest() [Fact] public async Task GetJsonGzipLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.Gzip; LogsContentType contentType = LogsContentType.Json; @@ -212,7 +212,7 @@ public async Task GetJsonGzipLogsTest() [Fact] public async Task GetTextGzipLogsTest() { - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.None(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.Gzip; LogsContentType contentType = LogsContentType.Text; diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs index b34f91b32a3..da32aaa92ea 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs @@ -31,15 +31,15 @@ public static IEnumerable GetLogsUploadRequestHandlerData() yield return new object[] { @"{""schemaVersion"":""1.0"",""sasUrl"":"""",""items"":{""id"":""edgeHub""},""encoding"":""none"",""contentType"":""json""}".Replace("", sasUrl), "edgeHub", sasUrl, LogsContentEncoding.None, LogsContentType.Json, ModuleLogFilter.Empty }; sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some(true), Option.Some("ERR")); yield return new object[] { @"{""schemaVersion"":""1.0"",""sasUrl"":"""",""items"":{""id"":""edgeAgent"",""filter"":}}".Replace("", sasUrl).Replace("", filter.ToJson()), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json, filter }; sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - filter = new ModuleLogFilter(Option.None(), Option.Some("1501000"), Option.None(), Option.None(), Option.Some("ERR")); + filter = new ModuleLogFilter(Option.None(), Option.Some("1501000"), Option.None(), Option.None(), Option.Some(true), Option.Some("ERR")); yield return new object[] { @"{""schemaVersion"":""1.5"",""sasUrl"":"""",""items"":{""id"":""edgeAgent"",""filter"":}}".Replace("", sasUrl).Replace("", filter.ToJson()), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json, filter }; sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - filter = new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.Some(3), Option.None()); + filter = new ModuleLogFilter(Option.Some(100), Option.None(), Option.None(), Option.Some(3), Option.Some(true), Option.None()); yield return new object[] { @"{""schemaVersion"":""1.2"",""sasUrl"":"""",""items"":{""id"":""edgeAgent"",""filter"":}}".Replace("", sasUrl).Replace("", filter.ToJson()), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json, filter }; } @@ -63,7 +63,7 @@ public async Task TestLogsUploadRequest(string payload, string id, string sasUrl } else { - logsProvider.Setup(l => l.GetLogs(id, moduleLogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(id, It.IsAny(), It.IsAny())) .ReturnsAsync(uploadBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, id, uploadBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); @@ -73,6 +73,7 @@ public async Task TestLogsUploadRequest(string payload, string id, string sasUrl { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, string.Empty, 0, Option.None(), Option.None()) }; + var runtimeInfoProvider = Mock.Of(r => r.GetModules(It.IsAny()) == Task.FromResult(moduleRuntimeInfoList)); // Act @@ -98,7 +99,8 @@ public async Task TestLogsUploadAllTaskRequest() { // Arrange string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; - var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some("ERR")); + // Note: When LogsContentType = Json, edgeAgent automatically set the "includeTimestamp" to be true. + var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None(), Option.Some(3), Option.Some(true), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Json; @@ -134,21 +136,21 @@ public async Task TestLogsUploadAllTaskRequest() var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None(), false); var mod1LogBytes = new byte[100]; - logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(mod1, It.IsAny(), It.IsAny())) .ReturnsAsync(mod1LogBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, mod1, mod1LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module2LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None(), false); var mod2LogBytes = new byte[80]; - logsProvider.Setup(l => l.GetLogs(mod2, module2LogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(mod2, It.IsAny(), It.IsAny())) .ReturnsAsync(mod2LogBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, mod2, mod2LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module3LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None(), false); var mod3LogBytes = new byte[120]; - logsProvider.Setup(l => l.GetLogs(mod3, module3LogOptions, It.IsAny())) + logsProvider.Setup(l => l.GetLogs(mod3, It.IsAny(), It.IsAny())) .ReturnsAsync(mod3LogBytes); logsUploader.Setup(l => l.UploadLogs(sasUrl, mod3, mod3LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs index 73511aa7226..2244041a3f4 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/RuntimeInfoProviderTest.cs @@ -285,7 +285,7 @@ public async Task GetModuleLogsTest() var runtimeInfoProvider = await RuntimeInfoProvider.CreateAsync(dockerClient); // Act - Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), CancellationToken.None); + Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), Option.None(), CancellationToken.None); // Assert Assert.NotNull(receivedContainerLogsParameters); @@ -301,7 +301,7 @@ public async Task GetModuleLogsTest() Assert.Equal(dummyLogs, receivedLogs); // Act - receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.Some(1000), Option.None(), Option.None(), CancellationToken.None); + receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.Some(1000), Option.None(), Option.None(), Option.None(), CancellationToken.None); // Assert Assert.NotNull(receivedContainerLogsParameters); @@ -317,7 +317,7 @@ public async Task GetModuleLogsTest() Assert.Equal(dummyLogs, receivedLogs); // Act - receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.None(), Option.Some("1552887267"), Option.None(), CancellationToken.None); + receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, true, Option.None(), Option.Some("1552887267"), Option.None(), Option.None(), CancellationToken.None); // Assert Assert.NotNull(receivedContainerLogsParameters); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs index 617b97313b8..647bb124857 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/ModuleManagementHttpClientTest.cs @@ -262,7 +262,7 @@ public async Task ModuleLogsTest(string serverApiVersion, string clientApiVersio IModuleManager client = new ModuleManagementHttpClient(this.serverUrl, serverApiVersion, clientApiVersion); // Act - Stream logsStream = await client.GetModuleLogs("edgeHub", false, Option.None(), Option.None(), Option.None(), CancellationToken.None); + Stream logsStream = await client.GetModuleLogs("edgeHub", false, Option.None(), Option.None(), Option.None(), Option.Some(false), CancellationToken.None); // Assert Assert.NotNull(logsStream); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs index c211cecdcf6..0498a172e60 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/RuntimeInfoProviderTest.cs @@ -101,13 +101,13 @@ public async Task GetLogsTest() Stream GetLogsStream() => new MemoryStream(Encoding.UTF8.GetBytes(dummyLogs)); var moduleManager = new Mock(); - moduleManager.Setup(m => m.GetModuleLogs(id, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny>(), It.IsAny())) + moduleManager.Setup(m => m.GetModuleLogs(id, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny>(), Option.None(), It.IsAny())) .ReturnsAsync(GetLogsStream); var runtimeInfoProvider = new RuntimeInfoProvider(moduleManager.Object); // Act - Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), CancellationToken.None); + Stream receivedLogsStream = await runtimeInfoProvider.GetModuleLogs(id, false, Option.None(), Option.None(), Option.None(), Option.None(), CancellationToken.None); // Assert var buffer = new byte[1024]; diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs index 50e5c779d8c..c8108040ca8 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/stream/LogsStreamRequestHandlerTest.cs @@ -39,7 +39,7 @@ public async Task HandleTest() string id = "m1"; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), It.IsAny())) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), Option.None(), It.IsAny())) .ReturnsAsync(new MemoryStream(buffer)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); @@ -86,7 +86,7 @@ public async Task HandleWithCancellationTest() string id = "m1"; var runtimeInfoProvider = new Mock(); - runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), It.IsAny())) + runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None(), Option.None(), Option.None(), Option.None(), It.IsAny())) .ReturnsAsync(new MemoryStream(buffer)); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny())) .ReturnsAsync(new[] { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, "foo", 0, Option.None(), Option.None()) }); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs index ca135568966..f934805b955 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.IntegrationTest/DummyModuleManager.cs @@ -36,7 +36,7 @@ public class DummyModuleManager : IModuleManager public Task PrepareUpdateAsync(ModuleSpec moduleSpec) => throw new NotImplementedException(); - public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task GetModuleLogs(string name, bool follow, Option tail, Option since, Option until, Option includeTimestamp, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task GetSupportBundle(Option since, Option until, Option iothubHostname, Option edgeRuntimeOnly, CancellationToken token) => throw new NotImplementedException(); } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs b/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs index 8362cf2168a..db0a1618d9c 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs @@ -59,7 +59,7 @@ await this.runtime.DeployConfigurationAsync( string since = DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd'T'HH:mm:sZ"); string until = DateTime.Now.AddDays(+1).ToString("yyyy-MM-dd'T'HH:mm:sZ"); - var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.Some(10), Option.Some(since), Option.Some(until), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); + var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.Some(10), Option.Some(since), Option.Some(until), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); var result = await this.IotHub.InvokeMethodAsync(this.runtime.DeviceId, ConfigModuleName.EdgeAgent, new CloudToDeviceMethod("GetModuleLogs", TimeSpan.FromSeconds(300), TimeSpan.FromSeconds(300)).SetPayloadJson(JsonConvert.SerializeObject(request)), token); @@ -88,7 +88,7 @@ await this.runtime.DeployConfigurationAsync( Context.Current.NestedEdge); await Task.Delay(30000); - var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); + var request = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); var result = await this.IotHub.InvokeMethodAsync(this.runtime.DeviceId, ConfigModuleName.EdgeAgent, new CloudToDeviceMethod("GetModuleLogs", TimeSpan.FromSeconds(300), TimeSpan.FromSeconds(300)).SetPayloadJson(JsonConvert.SerializeObject(request)), token); @@ -126,7 +126,7 @@ await this.runtime.DeployConfigurationAsync( await Task.Delay(10000); // check it restarted - var logsRequest = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); + var logsRequest = new ModuleLogsRequest("1.0", new List { new LogRequestItem(moduleName, new ModuleLogFilter(Option.None(), Option.None(), Option.None(), Option.None(), Option.None(), Option.None())) }, LogsContentEncoding.None, LogsContentType.Text); result = await this.IotHub.InvokeMethodAsync(this.runtime.DeviceId, ConfigModuleName.EdgeAgent, new CloudToDeviceMethod("GetModuleLogs", TimeSpan.FromSeconds(300), TimeSpan.FromSeconds(300)).SetPayloadJson(JsonConvert.SerializeObject(logsRequest)), token); Assert.AreEqual((int)HttpStatusCode.OK, result.Status);