Merge pull request #47 from Azure-Samples/optimize-indexing

Updating retry logic for V11 sdk
Azure-Samples · Oct 12, 2020 · d814447 · d814447
2 parents 6a55138 + 3fb6173
commit d814447
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 7 deletions.
diff --git a/optimize-data-indexing/v11/OptimizeDataIndexing/ExponentialBackoff.cs b/optimize-data-indexing/v11/OptimizeDataIndexing/ExponentialBackoff.cs
@@ -3,6 +3,7 @@
 using Azure.Search.Documents.Models;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 
 namespace OptimizeDataIndexing
@@ -14,6 +15,8 @@ private static async Task<IndexDocumentsResult> ExponentialBackoffAsync(SearchCl
         {
             // Create batch of documents for indexing
             var batch = IndexDocumentsBatch.Upload(hotels);
+
+            // Create an object to hold the result
             IndexDocumentsResult result = null;
 
             // Define parameters for exponential backoff
@@ -28,12 +31,43 @@ private static async Task<IndexDocumentsResult> ExponentialBackoffAsync(SearchCl
                 {
                     attempts++;
                     result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
+
+                    var failedDocuments = result.Results.Where(r => r.Succeeded != true).ToList();
+
+                    // handle partial failure
+                    if (failedDocuments.Count > 0)
+                    {
+
+                        if (attempts == maxRetryAttempts)
+                        {
+                            Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id);
+                            break;
+                        } 
+                        else
+                        {
+                            Console.WriteLine("[Batch starting at doc {0} had partial failure]", id);
+                            //Console.WriteLine("[Attempt: {0} of {1} Failed]", attempts, maxRetryAttempts);
+                            Console.WriteLine("[Retrying {0} failed documents] \n", failedDocuments.Count);
+
+                            // creating a batch of failed documents to retry
+                            var failedDocumentKeys = failedDocuments.Select(doc => doc.Key).ToList();
+                            hotels = hotels.Where(h => failedDocumentKeys.Contains(h.HotelId)).ToList();
+                            batch = IndexDocumentsBatch.Upload(hotels);
+
+                            Task.Delay(delay).Wait();
+                            delay = delay * 2;
+                            continue;
+                        }
+                    }
+
+
                     return result;
                 }
                 catch (RequestFailedException ex)
                 {
-                    Console.WriteLine("BATCH STARTING AT DOC {0}:", id);
-                    Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message);
+                    Console.WriteLine("[Batch starting at doc {0} failed]", id);
+                    //Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message);
+                    Console.WriteLine("[Retrying entire batch] \n");
 
                     if (attempts == maxRetryAttempts)
                     {

diff --git a/optimize-data-indexing/v11/OptimizeDataIndexing/Program.cs b/optimize-data-indexing/v11/OptimizeDataIndexing/Program.cs
@@ -40,7 +40,7 @@ public static async Task Main(string[] args)
 
             //long numDocuments = 100000;
             //DataGenerator dg = new DataGenerator();
-            //List<Hotel> hotels = dg.GetHotels(numDocuments);
+            //List<Hotel> hotels = dg.GetHotels(numDocuments, "large");
 
             //Console.WriteLine("{0}", "Uploading using exponential backoff...\n");
             //await ExponentialBackoff.IndexDataAsync(searchClient, hotels, 1000, 8);
@@ -71,8 +71,8 @@ private static async Task CreateIndexAsync(string indexName, SearchIndexClient i
             // Create a new search index structure that matches the properties of the Hotel class.
             // The Address class is referenced from the Hotel class. The FieldBuilder
             // will enumerate these to create a complex data structure for the index.
-            FieldBuilder bulder = new FieldBuilder();
-            var definition = new SearchIndex(indexName, bulder.Build(typeof(Hotel)));
+            FieldBuilder builder = new FieldBuilder();
+            var definition = new SearchIndex(indexName, builder.Build(typeof(Hotel)));
 
             await indexClient.CreateIndexAsync(definition);
         }

diff --git a/optimize-data-indexing/v11/OptimizeDataIndexing/appsettings.json b/optimize-data-indexing/v11/OptimizeDataIndexing/appsettings.json
@@ -1,5 +1,5 @@
 {
-  "SearchServiceUri": "<YourSearchServiceSearchServiceUri>",
-  "SearchServiceAdminApiKey": "<YourSearchServiceAdminApiKey>",
+  "SearchServiceUri": "https://{service-name}.search.windows.net",
+  "SearchServiceAdminApiKey": "",
   "SearchIndexName": "optimize-indexing"
 }