Skip to content

[RISE-3309] - Split migrations #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[RISE-3309] - Split migrations #35

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 155 additions & 20 deletions CategoryMigrationLambda/Services/CategoryMigrationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,28 +207,78 @@ private async Task ProcessDocumentsInBatches(
var oldCategoryId = preference.CategoryId;
var oldSubcategoryIds = preference.SubcategoryIds?.ToList() ?? new List<int>();

var (newCategoryId, newSubcategoryIds) = MigrateCategoryAndSubcategories(
var groupedMigrations = MigrateCategoryAndSubcategoriesGrouped(
preference.CategoryId, preference.SubcategoryIds ?? new List<int>());

if (groupedMigrations.Count == 0)
{
_logger.LogWarning("No valid migration found for preference {EntityId} with CategoryId {CategoryId}",
preference.EntityId, preference.CategoryId);
continue;
}

// Get the first (primary) category - this will update the existing preference
var primaryCategory = groupedMigrations.First();
var primaryCategoryId = primaryCategory.Key;
var primarySubcategoryIds = primaryCategory.Value;

if (!result.DryRun)
{
UpdateDocumentWithMigratedCategories(document, newCategoryId, newSubcategoryIds);
// Update the existing document with the primary category
UpdateDocumentWithMigratedCategories(document, primaryCategoryId, primarySubcategoryIds);
batch.Add(document);
}

result.MigratedCount++;

// Log the primary migration
if (userContext != null)
{
_logger.LogInformation("Migrated preference {EntityId} for user {UserContext}: CategoryId {OldCategoryId} -> {NewCategoryId}, SubcategoryIds [{OldSubcategoryIds}] -> [{NewSubcategoryIds}]",
preference.EntityId, userContext, oldCategoryId, newCategoryId,
string.Join(",", oldSubcategoryIds), string.Join(",", newSubcategoryIds));
preference.EntityId, userContext, oldCategoryId, primaryCategoryId,
string.Join(",", oldSubcategoryIds), string.Join(",", primarySubcategoryIds));
}
else
{
_logger.LogInformation("Migrated preference {EntityId}: CategoryId {OldCategoryId} -> {NewCategoryId}, SubcategoryIds [{OldSubcategoryIds}] -> [{NewSubcategoryIds}]",
preference.EntityId, oldCategoryId, newCategoryId,
string.Join(",", oldSubcategoryIds), string.Join(",", newSubcategoryIds));
preference.EntityId, oldCategoryId, primaryCategoryId,
string.Join(",", oldSubcategoryIds), string.Join(",", primarySubcategoryIds));
}

// Handle additional categories (create new preferences)
if (groupedMigrations.Count > 1)
{
var additionalCategories = groupedMigrations.Skip(1);

foreach (var additionalCategory in additionalCategories)
{
var newCategoryId = additionalCategory.Key;
var newSubcategoryIds = additionalCategory.Value;

if (!result.DryRun)
{
// Create a new document (preference) for this additional category
var newDocument = CreateNewPreferenceDocument(document, newCategoryId, newSubcategoryIds);
batch.Add(newDocument);
}

result.MigratedCount++;

// Log the new preference creation
var newEntityId = !result.DryRun ? document["EntityId"].AsString() + "_split_" + newCategoryId : "(dry-run)";
if (userContext != null)
{
_logger.LogInformation("Created NEW preference {NewEntityId} for user {UserContext} from {OriginalEntityId}: CategoryId -> {NewCategoryId}, SubcategoryIds -> [{NewSubcategoryIds}]",
newEntityId, userContext, preference.EntityId, newCategoryId,
string.Join(",", newSubcategoryIds));
}
else
{
_logger.LogInformation("Created NEW preference {NewEntityId} from {OriginalEntityId}: CategoryId -> {NewCategoryId}, SubcategoryIds -> [{NewSubcategoryIds}]",
newEntityId, preference.EntityId, newCategoryId,
string.Join(",", newSubcategoryIds));
}
}
}
Comment on lines +248 to 282
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Fix incorrect EntityId in log messages.

The logged newEntityId on line 268 doesn't match the actual EntityId created in CreateNewPreferenceDocument. Line 268 constructs a descriptive string (originalId_split_categoryId), but CreateNewPreferenceDocument generates a completely new GUID (line 346). This makes the logs misleading and could hinder debugging.

Apply this diff to log the actual EntityId:

                                if (!result.DryRun)
                                {
                                    // Create a new document (preference) for this additional category
                                    var newDocument = CreateNewPreferenceDocument(document, newCategoryId, newSubcategoryIds);
                                    batch.Add(newDocument);
+                                   var newEntityId = newDocument["EntityId"].AsString();
-                                   var newEntityId = !result.DryRun ? document["EntityId"].AsString() + "_split_" + newCategoryId : "(dry-run)";
                                    if (userContext != null)
                                    {
                                        _logger.LogInformation("Created NEW preference {NewEntityId} for user {UserContext} from {OriginalEntityId}: CategoryId -> {NewCategoryId}, SubcategoryIds -> [{NewSubcategoryIds}]",
                                            newEntityId, userContext, preference.EntityId, newCategoryId,
                                            string.Join(",", newSubcategoryIds));
                                    }
                                    else
                                    {
                                        _logger.LogInformation("Created NEW preference {NewEntityId} from {OriginalEntityId}: CategoryId -> {NewCategoryId}, SubcategoryIds -> [{NewSubcategoryIds}]",
                                            newEntityId, preference.EntityId, newCategoryId,
                                            string.Join(",", newSubcategoryIds));
                                    }
+                               }
+                               else
+                               {
+                                   var newEntityId = "(dry-run)";
+                                   if (userContext != null)
+                                   {
+                                       _logger.LogInformation("Would create NEW preference for user {UserContext} from {OriginalEntityId}: CategoryId -> {NewCategoryId}, SubcategoryIds -> [{NewSubcategoryIds}]",
+                                           userContext, preference.EntityId, newCategoryId,
+                                           string.Join(",", newSubcategoryIds));
+                                   }
+                                   else
+                                   {
+                                       _logger.LogInformation("Would create NEW preference from {OriginalEntityId}: CategoryId -> {NewCategoryId}, SubcategoryIds -> [{NewSubcategoryIds}]",
+                                           preference.EntityId, newCategoryId,
+                                           string.Join(",", newSubcategoryIds));
+                                   }
                                }

Committable suggestion skipped: line range outside the PR's diff.


if (batch.Count >= _batchSize)
Expand Down Expand Up @@ -281,6 +331,63 @@ private async Task WriteBatch(Table table, List<Document> batch, MigrationResult
}
}

private Document CreateNewPreferenceDocument(Document originalDocument, int? newCategoryId, List<int> newSubcategoryIds)
{
// Clone the original document to preserve all fields
var newDocument = new Document();

foreach (var key in originalDocument.Keys)
{
newDocument[key] = originalDocument[key];
}

// Generate a new unique EntityId
var originalEntityId = originalDocument["EntityId"].AsString();
var newEntityId = Guid.NewGuid().ToString();
newDocument["EntityId"] = newEntityId;

// Update SK to reflect the new EntityId
// SK format: ENTITYID#{EntityId}
if (originalDocument.ContainsKey("SK"))
{
var originalSK = originalDocument["SK"].AsString();
// Replace the old EntityId in SK with the new one
newDocument["SK"] = originalSK.Replace(originalEntityId, newEntityId);
}

// Update timestamps
var now = DateTime.UtcNow;
var timestamp = now.ToString("o");

// Update Gsi1SK to reflect the new EntityId and timestamp
// Gsi1SK format: TIMESTAMP#{timestamp}#ENTITYID#{EntityId}
if (originalDocument.ContainsKey("Gsi1SK"))
{
newDocument["Gsi1SK"] = $"TIMESTAMP#{timestamp}#ENTITYID#{newEntityId}";
}

// Update the migrated category and subcategories
newDocument["CategoryId"] = newCategoryId;

var subcategoryIdsList = new PrimitiveList(DynamoDBEntryType.Numeric);
foreach (var id in newSubcategoryIds)
{
subcategoryIdsList.Add(new Primitive(id.ToString(), true));
}
newDocument["SubcategoryIds"] = subcategoryIdsList;

// New split preferences should not be primary
if (originalDocument.ContainsKey("IsPrimary"))
{
newDocument["IsPrimary"] = false;
}

newDocument["CreatedAt"] = timestamp;
newDocument["UpdatedAt"] = timestamp;

return newDocument;
}

private void UpdateDocumentWithMigratedCategories(Document document, int? newCategoryId, List<int> newSubcategoryIds)
{
// Update the original document directly to preserve all fields
Expand All @@ -292,14 +399,16 @@ private void UpdateDocumentWithMigratedCategories(Document document, int? newCat
subcategoryIdsList.Add(new Primitive(id.ToString(), true));
}
document["SubcategoryIds"] = subcategoryIdsList;

// Update the timestamp
document["UpdatedAt"] = DateTime.UtcNow.ToString("o");
}

private (int? NewCategoryId, List<int> NewSubcategoryIds) MigrateCategoryAndSubcategories(
private Dictionary<int?, List<int>> MigrateCategoryAndSubcategoriesGrouped(
int? legacyCategoryId,
List<int>? legacySubcategoryIds)
{
var newSubcategoryIds = new List<int>();
int? newCategoryId = null;
var result = new Dictionary<int?, List<int>>();

// Handle null or empty subcategory list
if (legacySubcategoryIds == null || !legacySubcategoryIds.Any())
Expand All @@ -310,47 +419,73 @@ private void UpdateDocumentWithMigratedCategories(Document document, int? newCat
var categoryOnlyKey = (legacyCategoryId.Value, (int?)null);
if (CategoryMappings.CategoryMappingRulesByIds.TryGetValue(categoryOnlyKey, out var categoryOnlyMapping))
{
newCategoryId = categoryOnlyMapping.NewCategoryId;
if (!result.ContainsKey(categoryOnlyMapping.NewCategoryId))
{
result[categoryOnlyMapping.NewCategoryId] = new List<int>();
}
}
}
}
else
{
// Process each subcategory
// Process each subcategory and group by new category ID
foreach (var legacySubcategoryId in legacySubcategoryIds)
{
var mappingKey = (legacyCategoryId, legacySubcategoryId);

if (CategoryMappings.CategoryMappingRulesByIds.TryGetValue(mappingKey, out var mapping))
{
// Set the new category ID (should be consistent across all subcategories)
if (newCategoryId == null)
// Skip null mappings
if (!mapping.NewCategoryId.HasValue)
{
continue;
}

// Initialize list for this category if not exists
if (!result.ContainsKey(mapping.NewCategoryId))
{
newCategoryId = mapping.NewCategoryId;
result[mapping.NewCategoryId] = new List<int>();
}

// Add the new subcategory ID if it exists
if (mapping.NewSubcategoryId.HasValue)
{
newSubcategoryIds.Add(mapping.NewSubcategoryId.Value);
result[mapping.NewCategoryId].Add(mapping.NewSubcategoryId.Value);
}
}
}

// If no subcategory mappings were found, fall back to category-only mapping
if (newCategoryId == null && legacyCategoryId.HasValue)
if (!result.Any() && legacyCategoryId.HasValue)
{
var categoryOnlyKey = (legacyCategoryId.Value, (int?)null);
if (CategoryMappings.CategoryMappingRulesByIds.TryGetValue(categoryOnlyKey, out var categoryOnlyMapping))
{
newCategoryId = categoryOnlyMapping.NewCategoryId;
if (categoryOnlyMapping.NewCategoryId.HasValue)
{
result[categoryOnlyMapping.NewCategoryId] = new List<int>();
}
}
}
}

// Remove duplicates and sort
newSubcategoryIds = newSubcategoryIds.Distinct().OrderBy(x => x).ToList();
// Remove duplicates and sort subcategories for each category
foreach (var categoryId in result.Keys.ToList())
{
result[categoryId] = result[categoryId].Distinct().OrderBy(x => x).ToList();
}

return result;
}

private (int? NewCategoryId, List<int> NewSubcategoryIds) MigrateCategoryAndSubcategories(
int? legacyCategoryId,
List<int>? legacySubcategoryIds)
{
var grouped = MigrateCategoryAndSubcategoriesGrouped(legacyCategoryId, legacySubcategoryIds);

return (newCategoryId, newSubcategoryIds);
// Return the first (primary) category mapping for backward compatibility
var firstEntry = grouped.FirstOrDefault();
return (firstEntry.Key, firstEntry.Value ?? new List<int>());
}
}