Support crumb and cookie to fetching data

src/Helpers/CrumbHelper.cs

@@ -0,0 +1,77 @@
+using System.Net.Http;
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("OoplesFinance.YahooFinanceAPI.Tests.Unit")]
+namespace OoplesFinance.YahooFinanceAPI.Helpers
+{
+    internal sealed class CrumbHelper
+    {
+        /// <summary>
+        ///  Crumb value for the Yahoo Finance API
+        /// </summary>
+        internal readonly string Crumb;
+
+        internal static HttpMessageHandler handler = new HttpClientHandler();
+        private static List<string> cookies = new List<string>();
+        private static CrumbHelper? _instance;
+        CrumbHelper()
+        {
+            Crumb = string.Empty;
+
+            HttpClient client = GetHttpClient();
+
+            var loginResponse = client.GetAsync("https://login.yahoo.com/").Result;            
+
+            if (loginResponse.IsSuccessStatusCode)
+            {
+                var login = loginResponse.Content.ReadAsStringAsync().Result;
+                if (loginResponse.Headers.TryGetValues(name: "Set-Cookie", out IEnumerable<string>? setCookie))
+                {
+                    cookies = setCookie.Where(c => c.ToLower().IndexOf("domain=.yahoo.com") != -1).ToList();                    
+                    var crumbResponse = client.GetAsync("https://query1.finance.yahoo.com/v1/test/getcrumb").Result;
+
+                    if (crumbResponse.IsSuccessStatusCode)
+                    {
+                        Crumb = crumbResponse.Content.ReadAsStringAsync().Result;
+                    }
+                }
+            }
+            if (string.IsNullOrEmpty(Crumb))
+            {
+                throw new Exception("Failed to get crumb");
+            }
+        }
+
+        internal void Destory()
+        {
+            _instance = null;
+        }
+
+        public HttpClient GetHttpClient()
+        {
+            HttpClient client = new HttpClient(handler);            
+            client.DefaultRequestHeaders.Add("Cookie", cookies);
+            client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
+            client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
+            client.DefaultRequestHeaders.Add("Connection", "keep-alive");
+            client.DefaultRequestHeaders.Add("Pragma", "no-cache");
+            client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
+            return client;
+        }
+
+        /// <summary>
+        /// Single instance of the CrumbHelper
+        /// </summary>        
+        public static CrumbHelper Instance
+        {
+            get
+            {
+                if (_instance == null)
+                {
+                    _instance = new CrumbHelper();
+                }
+                return _instance;
+            }
+        }
+    }
+}

src/Helpers/DownloadHelper.cs

@@ -87,9 +87,8 @@ private static async Task<string> DownloadRawDataAsync(string urlString)
         }
         else
         {
-            using var client = new HttpClient();
-            using var request = new HttpRequestMessage(HttpMethod.Get, urlString);
-            var response = await client.SendAsync(request);
+            using var client = CrumbHelper.Instance.GetHttpClient();   
+            var response = await client.GetAsync(urlString);
 
             if (response.IsSuccessStatusCode)
             {
@@ -98,6 +97,7 @@ private static async Task<string> DownloadRawDataAsync(string urlString)
             }
             else
             {
+                CrumbHelper.Instance.Destory();
                 throw response.StatusCode switch
                 {
                     // Handle failure

src/Helpers/UrlHelper.cs

@@ -14,7 +14,7 @@ internal static class UrlHelper
     /// <param name="includeAdjClose"></param>
     /// <returns></returns>
     internal static string BuildYahooCsvUrl(string symbol, DataType dataType, DataFrequency dataFrequency, DateTime startDate, DateTime? endDate, bool includeAdjClose) => 
-        string.Format(CultureInfo.InvariantCulture, $"https://query1.finance.yahoo.com/v7/finance/download/{symbol}?period1={startDate.ToUnixTimestamp()}" +
+        string.Format(CultureInfo.InvariantCulture, $"https://query2.finance.yahoo.com/v7/finance/download/{symbol}?period1={startDate.ToUnixTimestamp()}" +
             $"&period2={(endDate ?? DateTime.Now).ToUnixTimestamp()}&interval={GetFrequencyString(dataFrequency)}&events={GetEventsString(dataType)}" +
             $"&includeAdjustedClose={includeAdjClose}");
 
@@ -103,8 +103,7 @@ internal static string BuildYahooSparkChartUrl(IEnumerable<string> symbols, Time
     /// <param name="module"></param>
     /// <returns></returns>
     internal static string BuildYahooStatsUrl(string symbol, Country country, Language language, YahooModule module) =>
-        string.Format(CultureInfo.InvariantCulture, $"https://query1.finance.yahoo.com/v11/finance/quoteSummary/{symbol}?lang={GetLanguageString(language)}" +
-            $"&region={GetCountryString(country)}&modules={GetModuleString(module)}");
+        string.Format(CultureInfo.InvariantCulture, $"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={CrumbHelper.Instance.Crumb}&lang={GetLanguageString(language)}&region={GetCountryString(country)}&modules={GetModuleString(module)}");
 
     /// <summary>
     /// Creates a url that will be used to get real-time quotes for multiple symbols
@@ -114,8 +113,8 @@ internal static string BuildYahooStatsUrl(string symbol, Country country, Langua
     /// <param name="language"></param>
     /// <returns></returns>
     internal static string BuildYahooRealTimeQuoteUrl(IEnumerable<string> symbols, Country country, Language language) =>
-        string.Format(CultureInfo.InvariantCulture, $"https://query1.finance.yahoo.com/v6/finance/quote?region=" +
-            $"{GetCountryString(country)}&lang={GetLanguageString(language)}&symbols={GetSymbolsString(symbols)}");
+        string.Format(CultureInfo.InvariantCulture, $"https://query2.finance.yahoo.com/v7/finance/quote?region=" +
+            $"{GetCountryString(country)}&lang={GetLanguageString(language)}&symbols={GetSymbolsString(symbols)}&crumb={CrumbHelper.Instance.Crumb}");
 
     /// <summary>
     /// Returns a custom string for the symbols option

src/Models/TrendingStocksData.cs

@@ -85,7 +85,7 @@ public record TrendingStocksQuote(
     [property: JsonProperty("epsForward", NullValueHandling = NullValueHandling.Ignore)] double? EpsForward,
     [property: JsonProperty("epsCurrentYear", NullValueHandling = NullValueHandling.Ignore)] double? EpsCurrentYear,
     [property: JsonProperty("priceEpsCurrentYear", NullValueHandling = NullValueHandling.Ignore)] double? PriceEpsCurrentYear,
-    [property: JsonProperty("sharesOutstanding", NullValueHandling = NullValueHandling.Ignore)] int? SharesOutstanding,
+    [property: JsonProperty("sharesOutstanding", NullValueHandling = NullValueHandling.Ignore)] long? SharesOutstanding,
     [property: JsonProperty("bookValue", NullValueHandling = NullValueHandling.Ignore)] double? BookValue,
     [property: JsonProperty("fiftyDayAverage", NullValueHandling = NullValueHandling.Ignore)] double? FiftyDayAverage,
     [property: JsonProperty("fiftyDayAverageChange", NullValueHandling = NullValueHandling.Ignore)] double? FiftyDayAverageChange,

src/OoplesFinance.YahooFinanceAPI.csproj

@@ -133,6 +133,9 @@
     <EmbeddedResource Remove="src\**" />
     <None Remove="Images\**" />
     <None Remove="src\**" />
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>OoplesFinance.YahooFinanceAPI.Tests.Unit</_Parameter1>
+    </AssemblyAttribute>
   </ItemGroup>
 
   <ItemGroup>

src/YahooClient.cs

@@ -569,9 +569,9 @@ public async Task<IEnumerable<SparkInfo>> GetSparkChartInfoAsync(IEnumerable<str
     /// </summary>
     /// <param name="symbol"></param>
     /// <returns></returns>
-    public async Task<RealTimeQuoteResult> GetRealTimeQuotesAsync(string symbol)
+    public async Task<RealTimeQuoteResult?> GetRealTimeQuotesAsync(string symbol)
     {
-        return new RealTimeQuoteHelper().ParseYahooJsonData<RealTimeQuoteResult>(await DownloadRealTimeQuoteDataAsync(symbol, Country, Language)).First();
+        return new RealTimeQuoteHelper().ParseYahooJsonData<RealTimeQuoteResult>(await DownloadRealTimeQuoteDataAsync(symbol, Country, Language)).FirstOrDefault();
     }
 
     /// <summary>

tests/UnitTests/OoplesFinance.YahooFinanceAPI.Tests.Unit.csproj

@@ -15,6 +15,7 @@
     </PackageReference>
     <PackageReference Include="FluentAssertions" Version="6.12.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
+    <PackageReference Include="Moq.Contrib.HttpClient" Version="1.4.0" />
     <PackageReference Include="NSubstitute" Version="5.1.0" />
     <PackageReference Include="xunit" Version="2.6.6" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6">

tests/UnitTests/YahooClientTests.cs

@@ -1,10 +1,15 @@
+using Moq;
+using Moq.Contrib.HttpClient;
+using System.Net;
+
 namespace OoplesFinance.YahooFinanceAPI.Tests.Unit;
 
 public sealed class YahooClientTests
 {
     private readonly YahooClient _sut;
     private const string BadSymbol = "OOPLES";
     private const string GoodSymbol = "MSFT";
+    private const string GoodFundSymbol = "VTSAX";
     private const int ValidCount = 10;
     private const int InvalidCount = -1;
     private readonly DateTime _startDate;
@@ -15,10 +20,10 @@ public sealed class YahooClientTests
     public YahooClientTests()
     {
         _sut = new YahooClient();
-        _startDate = DateTime.Now.AddYears(-1);
+        _startDate = DateTime.Now.AddMonths(-1);
         _emptySymbols = Enumerable.Empty<string>();
         _tooManySymbols = Enumerable.Repeat(GoodSymbol, 255);
-        _goodSymbols = Enumerable.Repeat(GoodSymbol, 50);
+        _goodSymbols = Enumerable.Repeat(GoodSymbol, 20);
     }
 
     [Fact]
@@ -183,10 +188,10 @@ public async Task GetTopTrendingStocks_ReturnsData_WhenCountIsValid()
         // Arrange
 
         // Act
-        var result = async () => await _sut.GetTopTrendingStocksAsync(Country.UnitedStates, ValidCount);
+        var result = await _sut.GetTopTrendingStocksAsync(Country.UnitedStates, ValidCount);
 
         // Assert
-        await result.Should().ThrowAsync<ArgumentException>().WithMessage("Count Must Be At Least 1 To Return Any Data");
+        result.Should().NotBeNull();
     }
 
     [Fact]
@@ -903,7 +908,7 @@ public async Task GetFundProfile_ReturnsData_WhenValidSymbolIsUsed()
         // Arrange
 
         // Act
-        var result = await _sut.GetFundProfileAsync(GoodSymbol);
+        var result = await _sut.GetFundProfileAsync(GoodFundSymbol);
 
         // Assert
         result.Should().NotBeNull();
@@ -1455,10 +1460,10 @@ public async Task GetRealTimeQuotes_ThrowsException_WhenNoSymbolIsFound()
         // Arrange
 
         // Act
-        var result = async () => await _sut.GetRealTimeQuotesAsync(BadSymbol);
+        var result = await _sut.GetRealTimeQuotesAsync(BadSymbol);
 
         // Assert
-        await result.Should().ThrowAsync<InvalidOperationException>().WithMessage("Requested Information Not Available On Yahoo Finance");
+        result.Should().BeNull();
     }
 
     [Fact]
@@ -2420,4 +2425,35 @@ public async Task GetTopUpsideBreakoutStocks_ReturnsData_WhenCountIsValid()
         // Assert
         result.Should().NotBeNull();
     }
+
+    [Fact]
+    public void CreateCrumbHelpInstnace_ReturnCrumb()
+    {
+        // Arrange
+
+        // Act
+        var crumbHelperInstance = OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.Instance;
+
+        // Assert
+        crumbHelperInstance.Crumb.Should().NotBeNullOrEmpty();
+    }
+
+    [Fact]
+    public void CreateCrumHelpInstance_ThrowsException_WhenFetchCrumbFailed()
+    {
+        // Arrange
+        var mockHandler = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+        mockHandler.SetupRequest(HttpMethod.Get, "https://login.yahoo.com/")
+            .ReturnsJsonResponse(HttpStatusCode.OK, "");
+        mockHandler.SetupRequest(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb")
+            .ReturnsJsonResponse(HttpStatusCode.OK,"");
+
+        //act
+        OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.handler = mockHandler.Object;
+        var ex = Record.Exception((() => OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.Instance));
+
+        //assert
+        ex.Message.Should().Be("Failed to get crumb");
+    }
+
 }