From 2d5a89a8747d9edd122974c4d60acc6f536b536a Mon Sep 17 00:00:00 2001 From: George Earl Date: Thu, 9 May 2024 19:24:38 +0100 Subject: [PATCH] chore: Refactor extract-content.py to include skip flags for re-running migration --- docs/deployment/migrate.md | 11 ++++++++++- scripts/extract-content.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/deployment/migrate.md b/docs/deployment/migrate.md index 3c6555a4b..d88ddf72b 100644 --- a/docs/deployment/migrate.md +++ b/docs/deployment/migrate.md @@ -31,4 +31,13 @@ make merge-databases ```bash make run-data-migration ``` -Once complete, your new deployment of 1.1 should be functional and contain the processed content from your 1.0 deployment. You can now plan to retire you old 1.0 deployment. \ No newline at end of file +Once complete, your new deployment of 1.1 should be functional and contain the processed content from your 1.0 deployment. You can now plan to retire you old 1.0 deployment. + +## Re-running a migration +If you encounter a failure while running the migration, you can re-run the process, but you have the option to skip steps to avoid repetition. You have the option of setting these values to True in the file called extract-content.py in the scripts folder: +```bash +skip_search_index = False +skip_cosmos_db = False +skip_upload_container = False +skip_content_container = False +``` \ No newline at end of file diff --git a/scripts/extract-content.py b/scripts/extract-content.py index 9c9e10690..5460bac20 100755 --- a/scripts/extract-content.py +++ b/scripts/extract-content.py @@ -7,6 +7,7 @@ from pyfiglet import Figlet import json import subprocess +import sys import os from azure.cosmos import CosmosClient, PartitionKey, exceptions from azure.identity import DefaultAzureCredential @@ -181,6 +182,10 @@ def get_storage_account_endpoint(storage_account_name): old_search_client = SearchClient(endpoint=old_search_endpoint, index_name=index_name, credential=AzureKeyCredential(old_search_key)) new_search_client = SearchClient(endpoint=new_search_endpoint, index_name=index_name, credential=AzureKeyCredential(new_search_key)) +error_guidance = 'If you re-run the process, you can skip sections that completed successfully by setting the corresponding skip flag to True. Read more details her' + + + # ************************************************************************* @@ -240,6 +245,7 @@ def get_storage_account_endpoint(storage_account_name): except Exception as e: print(e) + sys.exit() @@ -320,6 +326,7 @@ def get_storage_account_endpoint(storage_account_name): except exceptions.CosmosHttpResponseError as e: print(f'An error occurred: {e}') + sys.exit() # ************************************************************************* @@ -376,6 +383,7 @@ def get_storage_account_endpoint(storage_account_name): except Exception as e: print(e) + sys.exit() print(f"Completed migratiing upload container blobs. Processed {blobs_processed_count} blobs") @@ -417,6 +425,7 @@ def get_storage_account_endpoint(storage_account_name): except Exception as e: print(e) + sys.exit() print(f"Completed migratiing content container chnks. Processed {chunks_processed_count} blobs")