[INTERNAL] Client Telemetry: Adds shaded HdrHistogram Library (Azure#…

…2459)

As part of this PR, We are adding HDRHistogram support for Client Telemetry implementation where it will used to calculate different metrics.

Source
We are shading this library from https://github.com/HdrHistogram/HdrHistogram.NET

Legal
Made below changes to get approval from Legal team:

I have registered the component in CG, using cgmanifest.json, for security and legal approval. There are no alerts.
Copied the HdrHistogram license.txt in the repo.
Mentioned Creative Commons License in the header of each file.

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/HistogramEncoderV2.cs

@@ -0,0 +1,96 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+using System;
+using HdrHistogram.Utilities;
+
+namespace HdrHistogram.Encoding
+{
+    /// <summary>
+    /// An implementation of <see cref="IEncoder"/> for the V2 HdrHistogram log format.
+    /// </summary>
+    internal class HistogramEncoderV2 : IEncoder
+    {
+        /// <summary>
+        /// A singleton instance of the <see cref="HistogramEncoderV2"/>.
+        /// </summary>
+        public static readonly HistogramEncoderV2 Instance = new HistogramEncoderV2();
+
+
+        /// <summary>
+        /// Encodes the supplied <see cref="IRecordedData"/> into the supplied <see cref="ByteBuffer"/>.
+        /// </summary>
+        /// <param name="data">The data to encode.</param>
+        /// <param name="buffer">The target <see cref="ByteBuffer"/> to write to.</param>
+        /// <returns>The number of bytes written.</returns>
+        public int Encode(IRecordedData data, ByteBuffer buffer)
+        {
+            int initialPosition = buffer.Position;
+            buffer.PutInt(data.Cookie);
+            int payloadLengthPosition = buffer.Position;
+            buffer.PutInt(0); // Placeholder for payload length in bytes.
+            buffer.PutInt(data.NormalizingIndexOffset);
+            buffer.PutInt(data.NumberOfSignificantValueDigits);
+            buffer.PutLong(data.LowestDiscernibleValue);
+            buffer.PutLong(data.HighestTrackableValue);
+            buffer.PutDouble(data.IntegerToDoubleValueConversionRatio);
+
+            var payloadLength = FillBufferFromCountsArray(buffer, data);
+            buffer.PutInt(payloadLengthPosition, payloadLength);
+
+            var bytesWritten = buffer.Position - initialPosition;
+            return bytesWritten;
+        }
+
+        private static int FillBufferFromCountsArray(ByteBuffer buffer, IRecordedData data)
+        {
+            int startPosition = buffer.Position;
+            int srcIndex = 0;
+
+            while (srcIndex < data.Counts.Length)
+            {
+                // V2 encoding format uses a ZigZag LEB128-64b9B encoded long. 
+                // Positive values are counts, while negative values indicate a repeat zero counts. i.e. -4 indicates 4 sequential buckets with 0 counts.
+                long count = GetCountAtIndex(srcIndex++, data);
+                if (count < 0)
+                {
+                    throw new InvalidOperationException($"Cannot encode histogram containing negative counts ({count}) at index {srcIndex}");
+                }
+                // Count trailing 0s (which follow this count):
+                long zerosCount = 0;
+                if (count == 0)
+                {
+                    zerosCount = 1;
+                    while ((srcIndex < data.Counts.Length) && (GetCountAtIndex(srcIndex, data) == 0))
+                    {
+                        zerosCount++;
+                        srcIndex++;
+                    }
+                }
+                if (zerosCount > 1)
+                {
+                    ZigZagEncoding.PutLong(buffer, -zerosCount);
+                }
+                else
+                {
+                    ZigZagEncoding.PutLong(buffer, count);
+                }
+            }
+            return buffer.Position - startPosition;
+        }
+
+        private static long GetCountAtIndex(int idx, IRecordedData data)
+        {
+            return data.Counts[idx];
+            //var normalizedIdx = NormalizeIndex(idx, data.NormalizingIndexOffset, data.Counts.Length);
+            //return data.Counts[idx];
+        }
+    }
+}

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/IEncoder.cs

@@ -0,0 +1,28 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+using HdrHistogram.Utilities;
+
+namespace HdrHistogram.Encoding
+{
+    /// <summary>
+    /// Defines a method to allow histogram data to be encoded into a <see cref="ByteBuffer"/>.
+    /// </summary>
+    internal interface IEncoder
+    {
+        /// <summary>
+        /// Encodes the supplied <see cref="IRecordedData"/> into the supplied <see cref="ByteBuffer"/>.
+        /// </summary>
+        /// <param name="data">The data to encode.</param>
+        /// <param name="buffer">The target <see cref="ByteBuffer"/> to write to.</param>
+        /// <returns>The number of bytes written.</returns>
+        int Encode(IRecordedData data, ByteBuffer buffer);
+    }
+}

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/IHeader.cs

@@ -0,0 +1,51 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+namespace HdrHistogram.Encoding
+{
+    /// <summary>
+    /// Defines the header properties to be encoded for an HdrHistogram.
+    /// </summary>
+    internal interface IHeader
+    {
+        /// <summary>
+        /// The cookie value for the histogram.
+        /// </summary>
+        int Cookie { get; }
+        /// <summary>
+        /// The length in bytes of the payload body.
+        /// </summary>
+        int PayloadLengthInBytes { get; }
+        /// <summary>
+        /// The normalizing index offset.
+        /// </summary>
+        int NormalizingIndexOffset { get; } //Not currently implemented/used.
+        /// <summary>
+        /// THe number of significant digits that values are measured to.
+        /// </summary>
+        int NumberOfSignificantValueDigits { get; }
+        /// <summary>
+        /// The lowest trackable value for the histogram
+        /// </summary>
+        long LowestTrackableUnitValue { get; }
+        /// <summary>
+        /// The highest trackable value for the histogram
+        /// </summary>
+        long HighestTrackableValue { get; }
+        /// <summary>
+        /// Integer to double conversion ratio.
+        /// </summary>
+        double IntegerToDoubleValueConversionRatio { get; } //Not currently implemented/used.
+        /// <summary>
+        /// The amount of excess capacity that will not be needed.
+        /// </summary>
+        int CapacityEstimateExcess { get; }
+    }
+}

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/IRecordedData.cs

@@ -0,0 +1,47 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+namespace HdrHistogram.Encoding
+{
+    /// <summary>
+    /// Defines the histogram data to be recorded
+    /// </summary>
+    internal interface IRecordedData
+    {
+        /// <summary>
+        /// The cookie value for the histogram.
+        /// </summary>
+        int Cookie { get; }
+        /// <summary>
+        /// The normalizing index offset.
+        /// </summary>
+        int NormalizingIndexOffset { get; } //Required? What is it?
+        /// <summary>
+        /// THe number of significant digits that values are measured to.
+        /// </summary>
+        int NumberOfSignificantValueDigits { get; }
+        /// <summary>
+        /// The lowest trackable value for the histogram
+        /// </summary>
+        long LowestDiscernibleValue { get; }        //TODO: Use either LowestDiscernibleValue or LowestTrackableUnitValue but not both. -LC
+        /// <summary>
+        /// The highest trackable value for the histogram
+        /// </summary>
+        long HighestTrackableValue { get; }
+        /// <summary>
+        /// Integer to double conversion ratio.
+        /// </summary>
+        double IntegerToDoubleValueConversionRatio { get; }
+        /// <summary>
+        /// The actual array of counts.
+        /// </summary>
+        long[] Counts { get; }
+    }
+}

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/RecordedData.cs

@@ -0,0 +1,34 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+namespace HdrHistogram.Encoding
+{
+    internal sealed class RecordedData : IRecordedData
+    {
+        public RecordedData(int cookie, int normalizingIndexOffset, int numberOfSignificantValueDigits, long lowestDiscernibleValue, long highestTrackableValue, double integerToDoubleValueConversionRatio, long[] counts)
+        {
+            Cookie = cookie;
+            NormalizingIndexOffset = normalizingIndexOffset;
+            NumberOfSignificantValueDigits = numberOfSignificantValueDigits;
+            LowestDiscernibleValue = lowestDiscernibleValue;
+            HighestTrackableValue = highestTrackableValue;
+            IntegerToDoubleValueConversionRatio = integerToDoubleValueConversionRatio;
+            Counts = counts;
+        }
+
+        public int Cookie { get; }
+        public int NormalizingIndexOffset { get; }
+        public int NumberOfSignificantValueDigits { get; }
+        public long LowestDiscernibleValue { get; }
+        public long HighestTrackableValue { get; }
+        public double IntegerToDoubleValueConversionRatio { get; }
+        public long[] Counts { get; }
+    }
+}

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/V0Header.cs

@@ -0,0 +1,36 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+using HdrHistogram.Utilities;
+
+namespace HdrHistogram.Encoding
+{
+    internal sealed class V0Header : IHeader
+    {
+        public V0Header(int cookie, ByteBuffer buffer)
+        {
+            Cookie = cookie;
+            NumberOfSignificantValueDigits = buffer.GetInt();
+            LowestTrackableUnitValue = buffer.GetLong();
+            HighestTrackableValue = buffer.GetLong();
+            PayloadLengthInBytes = int.MaxValue;
+            IntegerToDoubleValueConversionRatio = 1.0;
+            NormalizingIndexOffset = 0;
+        }
+        public int Cookie { get; }
+        public int PayloadLengthInBytes { get; }
+        public int NormalizingIndexOffset { get; }
+        public int NumberOfSignificantValueDigits { get; }
+        public long LowestTrackableUnitValue { get; }
+        public long HighestTrackableValue { get; }
+        public double IntegerToDoubleValueConversionRatio { get; }
+        public int CapacityEstimateExcess => 32;
+    }
+}

Microsoft.Azure.Cosmos/src/OSS/HdrHistogram/Encoding/V1Header.cs

@@ -0,0 +1,37 @@
+// ------------------------------------------------------------
+// The code in this repository code was written by Lee Campbell, as a
+// derived work from the original Java by Gil Tene of Azul Systems and
+// Michael Barker, and released to the public domain, as explained
+// at http://creativecommons.org/publicdomain/zero/1.0/
+// ------------------------------------------------------------
+
+// This file isn't generated, but this comment is necessary to exclude it from StyleCop analysis.
+// <auto-generated/>
+
+using HdrHistogram.Utilities;
+
+namespace HdrHistogram.Encoding
+{
+    internal sealed class V1Header : IHeader
+    {
+        public V1Header(int cookie, ByteBuffer buffer)
+        {
+            Cookie = cookie;
+            PayloadLengthInBytes = buffer.GetInt();
+            NormalizingIndexOffset = buffer.GetInt();
+            NumberOfSignificantValueDigits = buffer.GetInt();
+            LowestTrackableUnitValue = buffer.GetLong();
+            HighestTrackableValue = buffer.GetLong();
+            IntegerToDoubleValueConversionRatio = buffer.GetDouble();
+        }
+
+        public int Cookie { get; }
+        public int PayloadLengthInBytes { get; }
+        public int NormalizingIndexOffset { get; }
+        public int NumberOfSignificantValueDigits { get; }
+        public long LowestTrackableUnitValue { get; }
+        public long HighestTrackableValue { get; }
+        public double IntegerToDoubleValueConversionRatio { get; }
+        public int CapacityEstimateExcess => 0;
+    }
+}