Skip to content

Commit

Permalink
Merge pull request Azure#3 from carjackson-msft/online-container-copy
Browse files Browse the repository at this point in the history
Online container copy
  • Loading branch information
niteshvijay1995 authored Feb 25, 2024
2 parents c873cf5 + bd63d86 commit a1140ef
Show file tree
Hide file tree
Showing 9 changed files with 2,284 additions and 679 deletions.
2 changes: 2 additions & 0 deletions src/cosmosdb-preview/HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ Release History
===============
0.27.0
* Add support for Per-Region Per-Partition Autoscale. '--enable-per-region-per-partition-autoscale' parameter can be used during account create/update.
* Add support for online mode in container copy job.

++++++
0.26.0
* Add '--enable-priority-based-execution' and '--default-priority-level' parameter for create/update database account.
Expand Down
9 changes: 9 additions & 0 deletions src/cosmosdb-preview/azext_cosmosdb_preview/_help.py
Original file line number Diff line number Diff line change
Expand Up @@ -884,6 +884,15 @@
az cosmosdb copy cancel -g "rg1" --account-name "acc1" --job-name "j1"
"""

helps['cosmosdb copy complete'] = """
type: command
short-summary: "Completes an online container copy job."
examples:
- name: Complete job j1
text: |-
az cosmosdb copy complete -g "rg1" --account-name "acc1" --job-name "j1"
"""

helps['cosmosdb sql container merge'] = """
type: command
short-summary: "Merges the partitions of a sql container."
Expand Down
7 changes: 5 additions & 2 deletions src/cosmosdb-preview/azext_cosmosdb_preview/_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,21 +465,24 @@ def load_arguments(self, _):
c.argument('dest_nosql', nargs='+', arg_group='Azure Cosmos DB API for NoSQL container copy', action=AddSqlContainerAction, help='Destination NoSql container details')
c.argument('host_copy_on_src', arg_type=get_three_state_flag(), help=argparse.SUPPRESS)
c.argument('worker_count', type=int, help=argparse.SUPPRESS)
c.argument('mode', help='Copy Mode (Online / Offline)')

for scope in [
'cosmosdb copy list',
'cosmosdb copy show',
'cosmosdb copy pause',
'cosmosdb copy resume',
'cosmosdb copy cancel']:
'cosmosdb copy cancel',
'cosmosdb copy complete']:
with self.argument_context(scope) as c:
c.argument('account_name', options_list=["--account-name", "-a"], id_part=None, required=True, help='Azure Cosmos DB account name where the job is created. Use --dest-account value from create job command.')

for scope in [
'cosmosdb copy show',
'cosmosdb copy pause',
'cosmosdb copy resume',
'cosmosdb copy cancel']:
'cosmosdb copy cancel',
'cosmosdb copy complete']:
with self.argument_context(scope) as c:
c.argument('job_name', options_list=['--job-name', '-n'], help='Name of the container copy job.', required=True)

Expand Down
1 change: 1 addition & 0 deletions src/cosmosdb-preview/azext_cosmosdb_preview/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ def load_command_table(self, _):
g.command('pause', 'pause')
g.command('resume', 'resume')
g.command('cancel', 'cancel')
g.command('complete', 'complete')

# Merge partitions for Sql containers
cosmosdb_sql_sdk = CliCommandType(
Expand Down
5 changes: 4 additions & 1 deletion src/cosmosdb-preview/azext_cosmosdb_preview/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -1612,7 +1612,8 @@ def cosmosdb_copy_job(client,
dest_mongo=None,
job_name=None,
worker_count=0,
host_copy_on_src=False):
host_copy_on_src=False,
mode="Offline"):
job_create_properties = {}
is_cross_account = src_account != dest_account
remote_account_name = dest_account if host_copy_on_src else src_account
Expand Down Expand Up @@ -1679,6 +1680,8 @@ def cosmosdb_copy_job(client,
if worker_count > 0:
job_create_properties['worker_count'] = worker_count

job_create_properties['mode'] = mode

job_create_parameters = {}
job_create_parameters['properties'] = job_create_properties

Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ def test_cosmosdb_copy_nosql(self, resource_group):
container_name = self.create_random_name(prefix='cli', length=15)
container_name_copied = self.create_random_name(prefix='cli', length=15)
job_name = self.create_random_name(prefix='cli', length=15)
remote_acc = self.create_random_name(prefix='cli', length=15)
remote_acc = self.create_random_name(prefix='cliremote', length=15)
cross_account_job_name = self.create_random_name(prefix='cli', length=15)
online_job_name = self.create_random_name(prefix='cli', length=15)
self.kwargs.update({
'acc': self.create_random_name(prefix='cli', length=15),
'remote_acc': remote_acc,
Expand All @@ -32,6 +33,7 @@ def test_cosmosdb_copy_nosql(self, resource_group):
'container_name_copied': container_name_copied,
'job_name': job_name,
'cross_account_job_name': cross_account_job_name,
'online_job_name': online_job_name,
'loc': 'eastus'
})

Expand Down Expand Up @@ -73,6 +75,12 @@ def test_cosmosdb_copy_nosql(self, resource_group):
assert job['jobName'] == job_name
assert job['status'] == "Cancelled"

self.cmd('az cosmosdb identity assign -n {acc} -g {rg}')
self.cmd('az cosmosdb update -n {acc} -g {rg} --default-identity="SystemAssignedIdentity"')

self.cmd('az cosmosdb create -n {remote_acc} -g {rg} --locations regionName={loc}')
account = self.cmd('az cosmosdb show -n {remote_acc} -g {rg}').get_output_in_json()

# Cross Account Copy Job
self.cmd('az cosmosdb copy create -g {rg} --job-name {cross_account_job_name} --src-account {remote_acc} --dest-account {acc} --src-nosql database={database_name} container={container_name} --dest-nosql database={database_name} container={container_name_copied}')

Expand All @@ -87,6 +95,23 @@ def test_cosmosdb_copy_nosql(self, resource_group):
assert job['destination']['databaseName'] == database_name
assert job['destination']['containerName'] == container_name_copied

# Enable Full Fidelity Change Feed
self.cmd('az resource update --name {acc} --resource-group {rg} --set properties.enableFullFidelityChangeFeed=true --resource-type databaseAccounts --namespace Microsoft.DocumentDB')

# Create online job
self.cmd('az cosmosdb copy create -g {rg} --mode online --job-name {online_job_name} --src-account {acc} --dest-account {acc} --src-nosql database={database_name} container={container_name} --dest-nosql database={database_name} container={container_name_copied}')

# Show job
job = self.cmd('az cosmosdb copy show -g {rg} --account-name {acc} --job-name {online_job_name}').get_output_in_json()
assert job['jobName'] == online_job_name
assert job['mode'] == 'Online'
assert job['source']['component'] == 'CosmosDBSql'
assert job['source']['databaseName'] == database_name
assert job['source']['containerName'] == container_name
assert job['destination']['component'] == 'CosmosDBSql'
assert job['destination']['databaseName'] == database_name
assert job['destination']['containerName'] == container_name_copied


@ResourceGroupPreparer(name_prefix='cli_test_cosmosdb_copy_mongo', location='eastus')
@AllowLargeResponse(size_kb=9999)
Expand Down

0 comments on commit a1140ef

Please sign in to comment.