Delay reading the HttpClient response until the ReceiveData property is accessed (#879)

* Delay reading the HttpClient response until the ReceiveData property is accessed. It allows chunked reading when the ReadChunk method is called.

* Rename _chunkedResponse to _response (the previous local variable 'response' is now an instance variable named '_response').
Rename ReadReponseData to ReadResponseData.

* Reset the _receiveData variable on HttpClientExecute; otherwise, a second execution on the same HttpClient instance may read the previous content.

* Disable compression in ASP.NET when streaming, as buffering is not supported alongside compression.

Author: claudiamurialdo

dotnet/DotNetStandardClasses.sln

@@ -249,6 +249,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAzureEventGrid", "src\dot
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCoreAttackMitigationTest", "test\DotNetCoreAttackMitigationTest\DotNetCoreAttackMitigationTest.csproj", "{2D615969-53E2-4B77-9A9A-75C33865CF76}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCoreChunkedTest", "test\DotNetCoreChunkedTest\DotNetCoreChunkedTest.csproj", "{5D2B1299-479F-430A-8D72-34D44FB299FD}"
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetPDFUnitTest", "test\DotNetPdfTest\DotNetPDFUnitTest.csproj", "{0FCFB078-5584-469F-92CC-61B0A6216D0D}"
 EndProject
 Global
@@ -605,6 +606,10 @@ Global
 		{2D615969-53E2-4B77-9A9A-75C33865CF76}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2D615969-53E2-4B77-9A9A-75C33865CF76}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2D615969-53E2-4B77-9A9A-75C33865CF76}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5D2B1299-479F-430A-8D72-34D44FB299FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5D2B1299-479F-430A-8D72-34D44FB299FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5D2B1299-479F-430A-8D72-34D44FB299FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5D2B1299-479F-430A-8D72-34D44FB299FD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0FCFB078-5584-469F-92CC-61B0A6216D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0FCFB078-5584-469F-92CC-61B0A6216D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0FCFB078-5584-469F-92CC-61B0A6216D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -728,6 +733,7 @@ Global
 		{5BBC75F0-E51A-4EBD-A628-92498D319B1D} = {4C43F2DA-59E5-46F5-B691-195449498555}
 		{7250CDB1-95C4-4822-B01B-3CBD73324CC9} = {30159B0F-BE61-4DB7-AC02-02851426BE4B}
 		{2D615969-53E2-4B77-9A9A-75C33865CF76} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC}
+		{5D2B1299-479F-430A-8D72-34D44FB299FD} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC}
 		{0FCFB078-5584-469F-92CC-61B0A6216D0D} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

dotnet/src/dotnetcore/GxClasses.Web/Properties/AssemblyInfo.cs

@@ -7,3 +7,4 @@
 [assembly: InternalsVisibleTo("GeneXus.Deploy.AzureFunctions.Handlers")]
 [assembly: InternalsVisibleTo("AzureFunctionsTest")]
 [assembly: InternalsVisibleTo("DotNetCoreAttackMitigationTest")]
+[assembly: InternalsVisibleTo("DotNetCoreChunkedTest")]

dotnet/src/dotnetcore/GxClasses/Properties/AssemblyInfo.cs

@@ -14,3 +14,4 @@
 [assembly: InternalsVisibleTo("GeneXus.Deploy.AzureFunctions.Handlers")]
 [assembly: InternalsVisibleTo("AzureFunctionsTest")]
 [assembly: InternalsVisibleTo("GXMessageBroker")]
+[assembly: InternalsVisibleTo("DotNetCoreChunkedTest")]

dotnet/src/dotnetframework/GxClasses/Core/Web/GxHttpServer.cs

@@ -148,7 +148,16 @@ public void AppendHeader( string name, string value)
 			{
 				value = GXUtil.EncodeContentDispositionHeader(value, _context.GetBrowserType());
 			}
-            if (_context!=null) 
+#if !NETCORE
+			else if (string.Compare(name,"Content-Type", true) == 0)
+			{
+				if (string.Compare(value, "text/event-stream", true) == 0)
+				{
+					_context.HttpContext.Response.BufferOutput = false;
+				}
+			}
+#endif
+			if (_context!=null) 
                 _context.SetHeader(name, value);
 		}
 

dotnet/src/dotnetframework/GxClasses/Domain/GxHttpClient.cs

@@ -77,6 +77,11 @@ public class GxHttpClient : IGxHttpClient, IDisposable
 		NameValueCollection _headers;
 		NameValueCollection _formVars;
 		MultiPartTemplate _multipartTemplate;
+		bool _isChunkedResponse;
+		HttpResponseMessage _response;
+		bool _eof;
+		bool _encodingFound;
+		string _charset;
 
 
 		string _scheme = "http://";
@@ -135,6 +140,7 @@ internal byte[] ReceiveData
 		{
 			get
 			{
+				ReadResponseData();
 				return _receiveData;
 			}
 		}
@@ -703,58 +709,40 @@ HttpResponseMessage ExecuteRequest(string method, string requestUrl, CookieConta
 					reqStream.Seek(0, SeekOrigin.Begin);
 					request.Content = new ByteArrayContent(reqStream.ToArray());
 					setHeaders(request, handler.CookieContainer);
-					response = client.SendAsync(request).GetAwaiter().GetResult();
+					response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).GetAwaiter().GetResult();
 				}
 			}
 			return response;
-		}
-		void ReadReponseContent(HttpResponseMessage response)
+		}		
+		void ReadResponseData()
 		{
-			_receiveData = Array.Empty<byte>();
-			try
+			if (_receiveData == null && _response!=null)
 			{
-				Stream stream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
-				string charset;
-				if (response.Content.Headers.ContentType == null)
-					charset = null;
-				else
-					charset = response.Content.Headers.ContentType.CharSet;
-				bool encodingFound = false;
-				if (!string.IsNullOrEmpty(charset))
+				_receiveData = Array.Empty<byte>();
+				try
 				{
-					int idx = charset.IndexOf("charset=");
-					if (idx > 0)
+					Stream stream = _response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
+
+					using (MemoryStream ms = new MemoryStream())
 					{
-						idx += 8;
-						charset = charset.Substring(idx, charset.Length - idx);
-						_encoding = GetEncoding(charset);
-						if (_encoding != null)
-							encodingFound = true;
+						stream.CopyTo(ms);
+						_receiveData = ms.ToArray();
 					}
-					else
+					_eof = true;
+					int bytesRead = _receiveData.Length;
+					GXLogging.Debug(log, "BytesRead " + _receiveData.Length);
+					if (bytesRead > 0 && !_encodingFound)
 					{
-						charset = String.Empty;
+						_encoding = DetectEncoding(_charset, out _encodingFound, _receiveData, bytesRead);
 					}
 				}
-				
-				using (MemoryStream ms = new MemoryStream())
+				catch (IOException ioEx)
 				{
-					stream.CopyTo(ms);
-					_receiveData = ms.ToArray();
+					if (_errCode == 1)
+						GXLogging.Warn(log, "Could not read response", ioEx);
+					else
+						throw ioEx;
 				}
-				int bytesRead = _receiveData.Length;
-				GXLogging.Debug(log, "BytesRead " + _receiveData.Length);
-				if (bytesRead > 0 && !encodingFound)
-				{
-					_encoding = DetectEncoding(charset, out encodingFound, _receiveData, bytesRead);
-				}				
-			}
-			catch (IOException ioEx)
-			{
-				if (_errCode == 1)
-					GXLogging.Warn(log, "Could not read response", ioEx);
-				else
-					throw ioEx;
 			}
 		}
 		bool UseOldHttpClient(string name)
@@ -790,10 +778,22 @@ public void Execute(string method, string name)
 				HttpClientExecute(method, name);
 			}
 		}
-
+		internal void ProcessResponse(HttpResponseMessage httpResponse)
+		{
+			_response = httpResponse;
+			LoadResponseHeaders(_response);
+			_statusCode = ((short)_response.StatusCode);
+			_statusDescription = GetStatusCodeDescrption(_response);
+			if ((_statusCode >= 400 && _statusCode < 600) && _errCode != 1)
+			{
+				_errCode = 1;
+				_errDescription = "The remote server returned an error: (" + _statusCode + ") " + _statusDescription + ".";
+			}
+		}
 		public void HttpClientExecute(string method, string name)
 		{
-			HttpResponseMessage response = null;
+			_receiveData = null;
+			_response = null;
 			Byte[] Buffer = new Byte[1024];
 			_errCode = 0;
 			_errDescription = string.Empty;
@@ -803,7 +803,7 @@ public void HttpClientExecute(string method, string name)
 				string requestUrl = GetRequestURL(name);
 				bool contextCookies = _context != null && !String.IsNullOrEmpty(requestUrl);
 				CookieContainer cookies = contextCookies ? _context.GetCookieContainer(requestUrl, IncludeCookies) : new CookieContainer();
-				response = ExecuteRequest(method, requestUrl, cookies);
+				_response = ExecuteRequest(method, requestUrl, cookies);
 
 				if (contextCookies)
 					_context.UpdateSessionCookieContainer();
@@ -819,9 +819,9 @@ public void HttpClientExecute(string method, string name)
 					_errDescription = aex.InnerException.Message;
 				else
 					_errDescription = aex.Message;
-				response = new HttpResponseMessage();
-				response.Content = new StringContent(HttpHelper.StatusCodeToTitle(HttpStatusCode.InternalServerError));
-				response.StatusCode = HttpStatusCode.InternalServerError;
+				_response = new HttpResponseMessage();
+				_response.Content = new StringContent(HttpHelper.StatusCodeToTitle(HttpStatusCode.InternalServerError));
+				_response.StatusCode = HttpStatusCode.InternalServerError;
 			}
 #endif
 			catch (HttpRequestException e)
@@ -832,22 +832,22 @@ public void HttpClientExecute(string method, string name)
 					_errDescription = e.Message + " " + e.InnerException.Message;
 				else
 					_errDescription = e.Message;
-				response = new HttpResponseMessage();
-				response.Content = new StringContent(HttpHelper.StatusCodeToTitle(HttpStatusCode.InternalServerError));
+				_response = new HttpResponseMessage();
+				_response.Content = new StringContent(HttpHelper.StatusCodeToTitle(HttpStatusCode.InternalServerError));
 #if NETCORE
-				response.StatusCode = (HttpStatusCode)(e.StatusCode != null ? e.StatusCode : HttpStatusCode.InternalServerError);
+				_response.StatusCode = (HttpStatusCode)(e.StatusCode != null ? e.StatusCode : HttpStatusCode.InternalServerError);
 #else
-				response.StatusCode = HttpStatusCode.InternalServerError;
+				_response.StatusCode = HttpStatusCode.InternalServerError;
 #endif
 			}
 			catch (TaskCanceledException e)
 			{
 				GXLogging.Warn(log, "Error Execute", e);
 				_errCode = 1;
 				_errDescription = "The request has timed out. " + e.Message;
-				response = new HttpResponseMessage();
-				response.StatusCode = 0;
-				response.Content = new StringContent(String.Empty);
+				_response = new HttpResponseMessage();
+				_response.StatusCode = 0;
+				_response.Content = new StringContent(String.Empty);
 			}
 			catch (Exception e)
 			{
@@ -857,29 +857,20 @@ public void HttpClientExecute(string method, string name)
 					_errDescription = e.Message + " " + e.InnerException.Message;
 				else
 					_errDescription = e.Message;
-				response = new HttpResponseMessage();
-				response.Content = new StringContent(HttpHelper.StatusCodeToTitle(HttpStatusCode.InternalServerError));
-				response.StatusCode = HttpStatusCode.InternalServerError;
+				_response = new HttpResponseMessage();
+				_response.Content = new StringContent(HttpHelper.StatusCodeToTitle(HttpStatusCode.InternalServerError));
+				_response.StatusCode = HttpStatusCode.InternalServerError;
 			}
 			GXLogging.Debug(log, "Reading response...");
-			if (response == null)
+			if (_response == null)
 				return;
-			LoadResponseHeaders(response);
-			ReadReponseContent(response);
-			_statusCode = ((short)response.StatusCode);
-			_statusDescription = GetStatusCodeDescrption(response);
-			if ((_statusCode >= 400 && _statusCode < 600) && _errCode != 1)
-			{
-				_errCode = 1;
-				_errDescription = "The remote server returned an error: (" + _statusCode + ") " + _statusDescription + ".";
-			}
+			ProcessResponse(_response);
 			ClearSendStream();
-			GXLogging.DebugSanitized(log, "_responseString " + ToString());
 		}
 		NameValueCollection _respHeaders;
 		private bool disposedValue;
 
-		void LoadResponseHeaders(HttpResponseMessage resp)
+		internal void LoadResponseHeaders(HttpResponseMessage resp)
 		{
 			_respHeaders = new NameValueCollection();
 			foreach (KeyValuePair<string, IEnumerable<string>> header in resp.Headers)
@@ -890,6 +881,29 @@ void LoadResponseHeaders(HttpResponseMessage resp)
 			{
 				_respHeaders.Add(header.Key, String.Join(",", header.Value));
 			}
+			_isChunkedResponse = resp.Headers.TransferEncodingChunked.HasValue && resp.Headers.TransferEncodingChunked.Value;
+
+			if (_response.Content.Headers.ContentType == null)
+				_charset = null;
+			else
+				_charset = _response.Content.Headers.ContentType.CharSet;
+			_encodingFound = false;
+			if (!string.IsNullOrEmpty(_charset))
+			{
+				int idx = _charset.IndexOf("charset=");
+				if (idx > 0)
+				{
+					idx += 8;
+					_charset = _charset.Substring(idx, _charset.Length - idx);
+					_encoding = GetEncoding(_charset);
+					if (_encoding != null)
+						_encodingFound = true;
+				}
+				else
+				{
+					_charset = String.Empty;
+				}
+			}
 		}
 
 		private string GetStatusCodeDescrption(HttpResponseMessage message)
@@ -1366,14 +1380,54 @@ private Encoding ExtractEncodingFromCharset(string responseText, string regExpP,
 			}
 			return enc;
 		}
+		public bool Eof
+		{
+			get
+			{
+				return _eof;
+
+			}
+		}
+		StreamReader _receivedChunkedStream;
+		public string ReadChunk()
+		{
+			if (!_isChunkedResponse)
+				return ToString();
+
+			if (_response == null)
+				return string.Empty;
+			try
+			{
+				if (_receivedChunkedStream == null)
+				{
+					_receivedChunkedStream = new StreamReader(_response.Content.ReadAsStreamAsync().GetAwaiter().GetResult());
+				}
+				_eof = _receivedChunkedStream.EndOfStream;
+				if (!_eof)
+				{
+					string line = _receivedChunkedStream.ReadLine();
+					if (line != null)
+					{
+						return line;
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				GXLogging.Error(log, String.Format("Error reading chunk", ex));
+			}
+			return string.Empty;
 
+		}
 		public override string ToString()
 		{
+			if (ReceiveData == null)
+				return string.Empty;
 			if (_encoding == null)
 				_encoding = Encoding.UTF8;
-			if (_receiveData == null)
-				return string.Empty;
-			return _encoding.GetString(_receiveData);
+			string responseString = _encoding.GetString(ReceiveData);
+			GXLogging.DebugSanitized(log, "_responseString " + responseString);
+			return responseString;
 		}
 		public void ToFile(string fileName)
 		{
@@ -1385,9 +1439,9 @@ public void ToFile(string fileName)
 			if (fileName.IndexOfAny(new char[] { '\\', ':' }) == -1)
 				pathName = Path.Combine(GxContext.StaticPhysicalPath(), fileName);
 
-			if (_receiveData != null)
+			if (ReceiveData != null)
 			{
-				File.WriteAllBytes(pathName, _receiveData);
+				File.WriteAllBytes(pathName, ReceiveData);
 			}
 		}
 
@@ -1463,6 +1517,7 @@ protected virtual void Dispose(bool disposing)
 				{
 					_receiveData = null;
 					_sendStream?.Dispose();
+					_receivedChunkedStream?.Dispose();
 				}
 				disposedValue = true;
 			}

dotnet/src/dotnetframework/GxClasses/Middleware/GXHttp.cs

@@ -2275,7 +2275,11 @@ public bool IsReusable
 
 		protected virtual void SetCompression(HttpContext httpContext)
 		{
+#if NETCORE
 			if (CompressHtmlResponse())
+#else
+			if (CompressHtmlResponse() && httpContext.Response.BufferOutput)
+#endif
 			{
 				GXUtil.SetGZip(httpContext);
 			}