Skip to content

Commit 68957d4

Browse files
sandeepsnairmssandeepsnairms
authored
Merge pull request #77 from sandeepsnairms/main
v0.7.15
2 parents 1e5ed6c + ebda6f4 commit 68957d4

File tree

9 files changed

+101
-33
lines changed

9 files changed

+101
-33
lines changed

‎MongoMigrationWebApp/Components/MigrationDetails.razor

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,21 @@
2929
</div>
3030
}
3131
<div class="mb-3">
32-
<label class="form-label">Online</label>
33-
<input type="checkbox" id="isOnline" disabled="@(!newMode)" @bind="isOnline" />
32+
<div style="display: flex; gap: 1rem; align-items: center;">
33+
<label class="form-label">
34+
<input type="checkbox" id="isOnline" disabled="@(!newMode)" @bind="isOnline" />
35+
Online
36+
</label>
37+
<label class="form-label">
38+
<input type="checkbox" id="copyIndex" disabled="@(!newMode)" @bind="appendMode" />
39+
Append Mode
40+
</label>
41+
<label class="form-label">
42+
<input type="checkbox" id="copyIndex" disabled="@(!newMode || appendMode)" @bind="skipIndexes" />
43+
Skip Indexes
44+
</label>
45+
46+
</div>
3447
</div>
3548

3649
<div class="mb-3">
@@ -49,8 +62,10 @@
4962
</div>
5063

5164
<div class="mb-3">
52-
<label for="isSimulatedRun" class="form-label">Simulation Mode: No Target Writes</label>
53-
<input type="checkbox" id="isSimulatedRun" disabled="@(!newMode)" @bind="isSimulatedRun" />
65+
<label for="isSimulatedRun" class="form-label">
66+
<input type="checkbox" id="isSimulatedRun" disabled="@(!newMode)" @bind="isSimulatedRun" />
67+
Simulation Mode (No Writes to Target)
68+
</label>
5469
</div>
5570

5671
@if (!string.IsNullOrEmpty(errorMessage))
@@ -68,7 +83,7 @@
6883

6984
@code {
7085

71-
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
86+
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
7287
#pragma warning disable CS8601
7388
#pragma warning disable CS8602
7489

@@ -100,6 +115,8 @@
100115
[Parameter]
101116
public Func<string, Task<bool>> CheckNameExists { get; set; }
102117

118+
private bool skipIndexes = false;// create indexes on Target
119+
private bool appendMode = false;// if the target collection exists, it will be deleted
103120
private string errorMessage = string.Empty;
104121
private bool useMongoDump = true; // Variable to track the selected value
105122
private string selectedOption = "Use MongoDump and MongoRestore"; // Default dropdown value
@@ -181,7 +198,9 @@
181198
TargetEndpoint = tmpTgtEndpoint,
182199
NameSpaces = namespaces,
183200
UseMongoDump = useMongoDump,
184-
IsSimulatedRun = isSimulatedRun
201+
IsSimulatedRun = isSimulatedRun,
202+
SkipIndexes=skipIndexes,
203+
AppendMode=appendMode
185204
};
186205
await OnSubmit.InvokeAsync(job);
187206
}

‎MongoMigrationWebApp/Shared/MainLayout.razor

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<main>
77
<div class="top-row px-4 d-flex justify-content-between align-items-center">
88
<h2 class="text-left">Migrate to Azure Cosmos DB for MongoDB (vCore based)</h2>
9-
<div class="ms-2 small">v0.7.14</div>
9+
<div class="ms-2 small">v0.7.15</div>
1010
</div>
1111

1212
<article class="content px-4">

‎MongoMigrationWebApp/readme.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
Version: 0.7.14
1+
Version: 0.7.15
22
Instructions: https://github.com/AzureCosmosDB/MongoMigrationWebBasedUtility
33
Purpose: An online and offline migration utility designed to migrate any MongoDB source to Azure Cosmos DB for Mongo vCore. It operates as an Azure Web App (or on-premises) and offers multiple methods for data migration. It can either use mongodump and mongorestore for data movement or employ the MongoDB driver to read data from the source and write it

‎OnlineMongoMigrationProcessor/Helpers/MongoHelper.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ public static async Task<bool> CheckCollectionExists(MongoClient client, string
271271
}
272272

273273

274-
public static async Task<bool> DeleteAndCopyIndexesAsync(string targetConnectionString, IMongoCollection<BsonDocument> sourceCollection)
274+
public static async Task<bool> DeleteAndCopyIndexesAsync(string targetConnectionString, IMongoCollection<BsonDocument> sourceCollection, bool skipIndexes)
275275
{
276276
try
277277
{
@@ -285,7 +285,7 @@ public static async Task<bool> DeleteAndCopyIndexesAsync(string targetConnection
285285
var targetDatabase = targetClient.GetDatabase(targetDatabaseName);
286286
var targetCollectionName = sourceCollectionName;
287287

288-
Log.WriteLine($"Creating collection with indexes: {targetDatabaseName}.{targetCollectionName}");
288+
Log.WriteLine($"Creating collection: {targetDatabaseName}.{targetCollectionName}");
289289
Log.Save();
290290

291291
// Check if the target collection exists
@@ -297,10 +297,16 @@ public static async Task<bool> DeleteAndCopyIndexesAsync(string targetConnection
297297
if (targetCollectionExists)
298298
{
299299
await targetDatabase.DropCollectionAsync(targetCollectionName);
300-
Log.WriteLine($"Deleted target collection: {targetDatabaseName}.{targetCollectionName}");
300+
Log.WriteLine($"Deleted existing target collection: {targetDatabaseName}.{targetCollectionName}");
301301
Log.Save();
302302
}
303303

304+
if (skipIndexes)
305+
return true;
306+
307+
Log.WriteLine($"Creating indexes for: {targetDatabaseName}.{targetCollectionName}");
308+
Log.Save();
309+
304310
// Get the indexes from the source collection
305311
var indexes = await sourceCollection.Indexes.ListAsync();
306312
var indexDocuments = await indexes.ToListAsync();

‎OnlineMongoMigrationProcessor/MigrationWorker.cs

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,13 +146,26 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
146146
{
147147
_sourceClient = new MongoClient(sourceConnectionString);
148148
Log.WriteLine("Source Client Created");
149-
if(job.IsSimulatedRun)
149+
if (job.IsSimulatedRun)
150150
{
151151
Log.WriteLine("Simulated Run. No changes will be made to the target.");
152-
}
152+
}
153+
else
154+
{
155+
if (job.AppendMode)
156+
{
157+
Log.WriteLine("Existing target collections will remain unchanged, and no indexes will be created.");
158+
}
159+
else
160+
{
161+
if (job.SkipIndexes)
162+
{
163+
Log.WriteLine("No indexes will be created.");
164+
}
165+
}
166+
}
153167
Log.Save();
154168

155-
156169
if (_job.IsOnline)
157170
{
158171
Log.WriteLine("Checking if Change Stream is enabled on source");
@@ -217,12 +230,12 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
217230

218231

219232

220-
if (!_job.UseMongoDump && !job.IsSimulatedRun)
233+
if (!_job.UseMongoDump && !job.IsSimulatedRun && !job.AppendMode)
221234
{
222235
var database = _sourceClient.GetDatabase(unit.DatabaseName);
223236
var collection = database.GetCollection<BsonDocument>(unit.CollectionName);
224-
await MongoHelper.DeleteAndCopyIndexesAsync(targetConnectionString, collection);
225-
}
237+
await MongoHelper.DeleteAndCopyIndexesAsync(targetConnectionString, collection, job.SkipIndexes);
238+
}
226239
}
227240

228241
}

‎OnlineMongoMigrationProcessor/Models/DataModels.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ public class MigrationJob
8989
public bool CurrentlyActive { get; set; }
9090
public bool UseMongoDump { get; set; }
9191
public bool IsSimulatedRun { get; set; }
92+
public bool SkipIndexes { get; set; }
93+
public bool AppendMode { get; set; }
9294
public List<MigrationUnit>? MigrationUnits { get; set; }
9395
}
9496

‎OnlineMongoMigrationProcessor/Processors/DumpRestoreProcessor.cs

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ public void Migrate(MigrationUnit item, string sourceConnectionString, string ta
141141
break;
142142
}
143143

144+
144145
if (item.MigrationChunks.Count > 1)
145146
{
146147
var bounds = SamplePartitioner.GetChunkBounds(item.MigrationChunks[i].Gte, item.MigrationChunks[i].Lt, item.MigrationChunks[i].DataType);
@@ -169,6 +170,7 @@ public void Migrate(MigrationUnit item, string sourceConnectionString, string ta
169170
docCount = item.MigrationChunks[i].DumpQueryDocCount;
170171
}
171172

173+
172174
if (Directory.Exists($"folder\\{i}.bson"))
173175
Directory.Delete($"folder\\{i}.bson", true);
174176

@@ -303,11 +305,20 @@ private void Upload(MigrationUnit item, string targetConnectionString)
303305
{
304306
string args = $" --uri=\"{targetConnectionString}\" --gzip {folder}\\{i}.bson";
305307

306-
// If first item, drop collection, else append
307-
if (i == 0)
308+
// If first item, drop collection, else append. Also No drop in AppendMode
309+
if (i == 0 && !_job.AppendMode)
310+
{
308311
args = $"{args} --drop";
312+
if (_job.SkipIndexes)
313+
{
314+
args = $"{args} --noIndexRestore"; // No index to create for all chunks.
315+
}
316+
}
309317
else
310-
args = $"{args} --noIndexRestore"; // No index for subsequent items.
318+
{
319+
args = $"{args} --noIndexRestore"; // No index to create. Index restore only for 1st chunk.
320+
321+
}
311322

312323
double initialPercent = ((double)100 / item.MigrationChunks.Count) * i;
313324
double contributionFactor = (double)item.MigrationChunks[i].DumpQueryDocCount / Math.Max(item.ActualDocCount, item.EstimatedDocCount);
@@ -392,7 +403,12 @@ private void Upload(MigrationUnit item, string targetConnectionString)
392403
}
393404
else
394405
{
395-
if (!_executionCancelled)
406+
if (item.MigrationChunks[i].IsUploaded == true)
407+
{
408+
continueProcessing = false;
409+
_jobs?.Save(); // Persist state
410+
}
411+
else if (!_executionCancelled)
396412
{
397413
Log.WriteLine($"Restore attempt {restoreAttempts} {dbName}.{colName}-{i} failed", LogType.Error);
398414
// Wait for the backoff duration before retrying

‎OnlineMongoMigrationProcessor/Processors/ProcessExecutor.cs

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Diagnostics;
44
using System.Text;
55
using System.Text.RegularExpressions;
6+
using System.Xml.Linq;
67

78
namespace OnlineMongoMigrationProcessor
89
{
@@ -163,20 +164,17 @@ private void ProcessErrorData(string data, string processType, MigrationUnit ite
163164
{
164165
if (processType == "MongoRestore")
165166
{
166-
var (restoredCount, failedCount) = ExtractRestoreCounts(data);
167+
var (restoredCount, failedCount, restorePercent) = ExtractRestoreCounts(data);
167168
if (restoredCount > 0 || failedCount > 0)
168169
{
169170
chunk.RestoredSuccessDocCount = restoredCount;
170171
chunk.RestoredFailedDocCount = failedCount;
171172
}
172-
}
173-
else
174-
{
175-
var dumpedDocCount = ExtractDumpedDocumentCount(data);
176-
if (dumpedDocCount > 0)
173+
if (restoredCount == 0 && failedCount == 0 && restorePercent ==100)
177174
{
178-
chunk.DumpResultDocCount = dumpedDocCount;
175+
chunk.IsUploaded = true;
179176
}
177+
180178
}
181179
if (!data.Contains("continuing through error: Duplicate key violation on the requested collection"))
182180
{
@@ -211,7 +209,7 @@ private string ExtractDocCount(string input)
211209

212210
}
213211

214-
public (int RestoredCount, int FailedCount) ExtractRestoreCounts(string input)
212+
public (int RestoredCount, int FailedCount, double percentage) ExtractRestoreCounts(string input)
215213
{
216214
// Regular expressions to capture the counts
217215
var restoredMatch = Regex.Match(input, @"(\d+)\s+document\(s\)\s+restored\s+successfully");
@@ -221,7 +219,19 @@ private string ExtractDocCount(string input)
221219
int restoredCount = restoredMatch.Success ? int.Parse(restoredMatch.Groups[1].Value) : 0;
222220
int failedCount = failedMatch.Success ? int.Parse(failedMatch.Groups[1].Value) : 0;
223221

224-
return (restoredCount, failedCount);
222+
double percentage=0;
223+
if (restoredCount==0 && failedCount==0)
224+
{
225+
var match = Regex.Match(input, @"\(([\d.]+)%\)");
226+
227+
if (match.Success)
228+
{
229+
percentage = double.Parse(match.Groups[1].Value);
230+
//Console.WriteLine($"Percentage: {percentage}%");
231+
}
232+
}
233+
234+
return (restoredCount, failedCount, percentage);
225235
}
226236

227237
public int ExtractDumpedDocumentCount(string input)

‎README.md

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -382,9 +382,11 @@ Follow these steps to migrate data from an on-premises MongoDB VM. You can deplo
382382
2. In the **New Job Details** pop-up, enter all required information.
383383
3. If necessary, use the [list collection steps](#create-comma-separated-list-of-collections) to create a comma-separated list of collection names.
384384
4. Choose the migration tool: either **Mongo Dump/Restore** or **Mongo Driver**.
385-
5. Select the desired [migration mode](#migrations-modes).
386-
6. Once all fields are filled, select **OK**.
387-
7. The job will automatically start if no other jobs are running.
385+
5. Select the desired [migration mode](#migrations-modes).
386+
6. Select **Append Mode** to preserve existing collection(s) on the target without deleting them.
387+
7. Select **Skip Indexes** to prevent the tool from copying indexes from the source.
388+
8. Once all fields are filled, select **OK**.
389+
9. The job will automatically start if no other jobs are running.
388390
389391
390392
**Note:** For the Mongo Dump/Restore option, the Web App will download the mongo-tools from the URL specified in the Web App settings. Ensure that the Web App has access to this URL. If the Web App does not have internet access, you can download the mongo-tools zip file to your development machine, then copy it to the wwwroot folder inside the published folder before compressing it. Afterward, update the URL in the Web App settings to point to the Web App’s URL (e.g., https://<WebAppName>.azurewebsites.net/<zipfilename.zip>).

0 commit comments

Comments
 (0)