Add support for chunked httpclient and httpresponse. (#823)

* Add support for chunked httpclient and chuncked httpresponse.

* Do not append headers for X-GXOBJECT and X-UA-Compatible.
X-GXOBJECT is set twice when called from SendHeaders and ValidateSpaRequest from webExecute.

* Condition content-disposition=inline; filename to reports only.

* Define new property EOF and method ReadChunk for HttpClient datatype.

* Delay adding the header X-SPA-MP until the MasterPageObj is instantiated.
Since sendSpaHeaders is called at the start of execution, the MasterPageObj is null at this moment.

* Define method ChunkedStreaming to set chunkedResponse at procedure level

* Setup Streaming after httpcontext is initialized

* Remove extra changes in spacing and indentation.

* _chunkedResponse condition depends only on TransferEncodingChunked header in the response.

* Use BufferOutput instead of the deprecated Buffer property in HttpResponse.

* Remove extra blank lines.

* ReceiveData was null when httpclient response is chunked.
Issue:103474

* Access SessionId at the beginning of Streamed https procs o to avoid HttpException: Session state has created a session id, but cannot save it because the response was already flushed by the application.

* Limit BringSessionStateToLife to .NET Framework.

Author: claudiamurialdo

dotnet/src/dotnetframework/GxClasses/Core/Web/GxHttpServer.cs

@@ -12,12 +12,6 @@ namespace GeneXus.Http.Server
 	using System.Linq;
 	using Microsoft.AspNetCore.Http.Features;
 	using System.Text;
-	using System.Threading.Tasks;
-	using Microsoft.AspNetCore.Mvc.Formatters;
-	using System.Net.Http;
-	using Stubble.Core.Contexts;
-	using System.Net.Mime;
-
 #endif
 
 	public class GxHttpCookie

dotnet/src/dotnetframework/GxClasses/Domain/GxHttpClient.cs

@@ -66,6 +66,7 @@ public class GxHttpClient : IGxHttpClient, IDisposable
 		const int StreamWriterDefaultBufferSize = 1024;
 		Stream _sendStream;
 		byte[] _receiveData;
+		StreamReader _receiveStream;
 		int _timeout = 30000;
 		short _statusCode = 0;
 		string _proxyHost;
@@ -133,6 +134,14 @@ internal byte[] ReceiveData
 		{
 			get
 			{
+				if (_chunkedResponse && _receiveData == null && _receiveStream!=null)
+				{
+					using (MemoryStream memstream = new MemoryStream())
+					{
+						_receiveStream.BaseStream.CopyTo(memstream);
+						_receiveData = memstream.ToArray();
+					}
+				}
 				return _receiveData;
 			}
 		}
@@ -414,6 +423,7 @@ public void AddHeader(string name, string value)
 					GXLogging.Error(log, String.Format("Error parsing charset ", value, ex));
 				}
 			}
+		
 			_headers.Set(name, value);
 		}
 		public void ClearVariables()
@@ -697,7 +707,7 @@ HttpResponseMessage ExecuteRequest(string method, string requestUrl, CookieConta
 					reqStream.Seek(0, SeekOrigin.Begin);
 					request.Content = new ByteArrayContent(reqStream.ToArray());
 					setHeaders(request, handler.CookieContainer);
-					response = client.SendAsync(request).GetAwaiter().GetResult();
+					response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).GetAwaiter().GetResult();
 				}
 			}
 			return response;
@@ -708,6 +718,7 @@ void ReadReponseContent(HttpResponseMessage response)
 			try
 			{
 				Stream stream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
+				_chunkedResponse = response.Headers.TransferEncodingChunked.HasValue && response.Headers.TransferEncodingChunked.Value;
 				string charset;
 				if (response.Content.Headers.ContentType == null)
 					charset = null;
@@ -730,18 +741,28 @@ void ReadReponseContent(HttpResponseMessage response)
 						charset = String.Empty;
 					}
 				}
-				
-				using (MemoryStream ms = new MemoryStream())
+				if (_chunkedResponse)
 				{
-					stream.CopyTo(ms);
-					_receiveData = ms.ToArray();
+					if (_encoding == null)
+						_encoding = Encoding.UTF8;
+
+					_receiveStream = new StreamReader(stream, _encoding);
+					_receiveData = null;
 				}
-				int bytesRead = _receiveData.Length;
-				GXLogging.Debug(log, "BytesRead " + _receiveData.Length);
-				if (bytesRead > 0 && !encodingFound)
+				else
 				{
-					_encoding = DetectEncoding(charset, out encodingFound, _receiveData, bytesRead);
-				}				
+					using (MemoryStream ms = new MemoryStream())
+					{
+						stream.CopyTo(ms);
+						_receiveData = ms.ToArray();
+					}
+					int bytesRead = _receiveData.Length;
+					GXLogging.Debug(log, "BytesRead " + _receiveData.Length);
+					if (bytesRead > 0 && !encodingFound)
+					{
+						_encoding = DetectEncoding(charset, out encodingFound, _receiveData, bytesRead);
+					}
+				}
 			}
 			catch (IOException ioEx)
 			{
@@ -787,7 +808,7 @@ public void Execute(string method, string name)
 
 		public void HttpClientExecute(string method, string name)
 		{
-			HttpResponseMessage response = null;
+			response = null;
 			Byte[] Buffer = new Byte[1024];
 			_errCode = 0;
 			_errDescription = string.Empty;
@@ -868,11 +889,11 @@ public void HttpClientExecute(string method, string name)
 				_errDescription = "The remote server returned an error: (" + _statusCode + ") " + _statusDescription + ".";
 			}
 			ClearSendStream();
-			GXLogging.DebugSanitized(log, "_responseString " + ToString());
 		}
 		NameValueCollection _respHeaders;
 		private bool disposedValue;
-
+		private bool _chunkedResponse;
+		HttpResponseMessage response;
 		void LoadResponseHeaders(HttpResponseMessage resp)
 		{
 			_respHeaders = new NameValueCollection();
@@ -1360,14 +1381,65 @@ private Encoding ExtractEncodingFromCharset(string responseText, string regExpP,
 			}
 			return enc;
 		}
+		public bool Eof
+		{
+			get
+			{
+				if (_chunkedResponse && _receiveStream != null)
+				{
+					return _receiveStream.EndOfStream;
+				}
+				return true;
+			}
+		}
 
-		public override string ToString()
+		public string ReadChunk()
 		{
-			if (_encoding == null)
-				_encoding = Encoding.UTF8;
-			if (_receiveData == null)
+			if (_chunkedResponse)
+			{
+				if (_receiveStream != null)
+				{
+					if (!_receiveStream.EndOfStream)
+					{
+						string line = _receiveStream.ReadLine();
+						if (line != null)
+							return line;
+					}
+					else
+					{
+						_receiveStream.Dispose();
+						_receiveStream = null;
+						response.Dispose();
+						response = null;
+					}
+				}
 				return string.Empty;
-			return _encoding.GetString(_receiveData);
+			}
+			else
+				return ToString();
+		}
+		public override string ToString()
+		{
+			string responseString;
+			if (_chunkedResponse)
+			{
+				StringBuilder sb = new StringBuilder();
+				while (!Eof){
+					sb.Append(ReadChunk());
+				}
+				responseString = sb.ToString();
+			}
+			else
+			{
+				if (_encoding == null)
+					_encoding = Encoding.UTF8;
+				if (_receiveData == null)
+					return string.Empty;
+
+				responseString = _encoding.GetString(_receiveData);
+			}
+			GXLogging.DebugSanitized(log, "_responseString " + responseString);
+			return responseString;
 		}
 		public void ToFile(string fileName)
 		{
@@ -1457,6 +1529,7 @@ protected virtual void Dispose(bool disposing)
 				{
 					_receiveData = null;
 					_sendStream?.Dispose();
+					_receiveStream?.Dispose();
 				}
 				disposedValue = true;
 			}

dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs

@@ -48,6 +48,13 @@ public class HttpHeader
 		public static string LAST_MODIFIED = "Last-Modified";
 		public static string EXPIRES = "Expires";
 		public static string XGXFILENAME = "x-gx-filename";
+		internal static string ACCEPT = "Accept";
+		internal static string TRANSFER_ENCODING = "Transfer-Encoding";
+	}
+	internal class HttpHeaderValue
+	{
+		internal static string ACCEPT_SERVER_SENT_EVENT = "text/event-stream";
+		internal static string TRANSFER_ENCODING_CHUNKED = "chunked";
 	}
 	[DataContract()]
 	public class HttpJsonError

dotnet/src/dotnetframework/GxClasses/Middleware/GXHttp.cs

@@ -1933,10 +1933,10 @@ public void ProcessRequest(HttpContext httpContext)
 			InitPrivates();
 			try
 			{
-#if NETCORE
+				SetStreaming();
 				SendHeaders();
 				string clientid = context.ClientID; //Send clientid cookie (before response HasStarted) if necessary, since UseResponseBuffering is not in .netcore3.0
-#endif
+
 				bool validSession = ValidWebSession();
 				if (validSession && IntegratedSecurityEnabled)
 					validSession = ValidSession();
@@ -1971,9 +1971,6 @@ public void ProcessRequest(HttpContext httpContext)
 						context.DispatchAjaxCommands();
 				}
 				SetCompression(httpContext);
-#if !NETCORE
-				SendHeaders();
-#endif
 				context.ResponseCommited = true;
 			}
 			catch (Exception e)
@@ -1988,6 +1985,25 @@ public void ProcessRequest(HttpContext httpContext)
 				throw new Exception("GXApplication exception", e);
 			}
 		}
+		protected virtual bool ChunkedStreaming() { return false; }
+
+		private void SetStreaming()
+		{
+#if !NETCORE
+			if (ChunkedStreaming())
+			{
+				BringSessionStateToLife();
+				context.HttpContext.Response.BufferOutput = false;
+			}
+#endif
+		}
+#if !NETCORE
+		private void BringSessionStateToLife()
+		{
+			string sessionId = context.HttpContext.Session.SessionID;
+			GXLogging.Debug(log, "Session is alive: " + sessionId);
+		}
+#endif
 		internal string DumpHeaders(HttpContext httpContext)
 		{
 			StringBuilder str = new StringBuilder();
@@ -2033,7 +2049,6 @@ private bool ValidWebSession()
 #else
 			bool isExpired = IsFullAjaxRequest(HttpContext.Current) && this.AjaxOnSessionTimeout() == "Warn" && GxWebSession.IsSessionExpired(localHttpContext);
 #endif
-
 			if (isExpired)
 			{
 				GXLogging.Info(log, "440 Session timeout. Web Session has expired and GX' OnSessionTimeout' Pty is set to 'WARN'");
@@ -2221,23 +2236,15 @@ public virtual void sendAdditionalHeaders()
 						return;
 
 					string safeIECompMode = IE_COMP_Edge.Equals(IE_COMP_EmulateIE7) ? IE_COMP_Edge : IE_COMP_Edge;
-#if NETCORE
 					localHttpContext.Response.Headers["X-UA-Compatible"] = "IE=" + safeIECompMode;
-#else
-					localHttpContext.Response.AddHeader("X-UA-Compatible", "IE=" + safeIECompMode);
-#endif
 				}
 			}
 
 		}
 
 		protected virtual void sendSpaHeaders()
 		{
-#if NETCORE
 			localHttpContext.Response.Headers[GX_SPA_GXOBJECT_RESPONSE_HEADER] = GetPgmname().ToLower();
-#else
-			localHttpContext.Response.AddHeader(GX_SPA_GXOBJECT_RESPONSE_HEADER, GetPgmname().ToLower());
-#endif
 		}
 
 		private void webExecuteWorker(object target)
@@ -3063,15 +3070,6 @@ public abstract class GXDataArea : GXHttpHandler
 
 		protected GXMasterPage MasterPageObj { get; set; }
 
-		protected override void sendSpaHeaders()
-		{
-			base.sendSpaHeaders();
-			if (MasterPageObj != null)
-			{
-				localHttpContext.Response.AddHeader(GX_SPA_MASTERPAGE_HEADER, MasterPageObj.GetPgmname());
-			}
-		}
-
 		protected override void ValidateSpaRequest()
 		{
 			string sourceMasterPage = localHttpContext.Request.Headers[GX_SPA_MASTERPAGE_HEADER];
@@ -3080,6 +3078,10 @@ protected override void ValidateSpaRequest()
 				context.DisableSpaRequest();
 				sendSpaHeaders();
 			}
+			if (MasterPageObj != null)
+			{
+				localHttpContext.Response.Headers[GX_SPA_MASTERPAGE_HEADER] = MasterPageObj.GetPgmname();
+			}
 		}
 	}
 	public class GXDataAreaControl

dotnet/src/dotnetframework/GxClasses/Model/GXWebProcedure.cs

@@ -8,6 +8,11 @@ namespace GeneXus.Procedure
 	using System.Threading;
 	using GeneXus.Mime;
 	using GeneXus.Utils;
+#if NETCORE
+	using Microsoft.AspNetCore.Http;
+#else
+	using System.Web;
+#endif
 	using log4net;
 	using GeneXus.Application;
 	using GeneXus.Data.NTier;
@@ -46,7 +51,13 @@ public override void cleanup()
 				context.DeleteReferer();
 			}
 		}
-
+		protected override void SetCompression(HttpContext httpContext)
+		{
+			if (!ChunkedStreaming())
+			{
+				base.SetCompression(httpContext);
+			}
+		}
 		public void setContextReportHandler()
 		{