Skip to content

Commit

Permalink
Fix Masking Indexes and Query (dataplat#6862)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanderstad
sanderstad authored Sep 24, 2020
1 parent 04f2c20 commit 0964968
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 38 deletions.
64 changes: 48 additions & 16 deletions functions/Invoke-DbaDbDataMasking.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,6 @@ function Invoke-DbaDbDataMasking {
.PARAMETER FilePath
Configuration file that contains the which tables and columns need to be masked
.PARAMETER Query
If you would like to mask only a subset of a table, use the Query parameter, otherwise all data will be masked.
.PARAMETER Locale
Set the local to enable certain settings in the masking
Expand Down Expand Up @@ -142,7 +139,6 @@ function Invoke-DbaDbDataMasking {
[string[]]$Column,
[string[]]$ExcludeTable,
[string[]]$ExcludeColumn,
[string]$Query,
[int]$MaxValue,
[int]$ModulusFactor = 10,
[switch]$ExactLength,
Expand Down Expand Up @@ -294,8 +290,9 @@ function Invoke-DbaDbDataMasking {

$dbTable = $db.Tables | Where-Object { $_.Schema -eq $tableobject.Schema -and $_.Name -eq $tableobject.Name }

$cleanupIdentityColumn = $false
[bool]$cleanupIdentityColumn = $false

# Make sure there is an identity column present to speed things up
if (-not ($dbTable.Columns | Where-Object Identity -eq $true)) {
Write-Message -Level Verbose -Message "Adding identity column to table [$($dbTable.Schema)].[$($dbTable.Name)]"
$query = "ALTER TABLE [$($dbTable.Schema)].[$($dbTable.Name)] ADD MaskingID BIGINT IDENTITY(1, 1) NOT NULL;"
Expand All @@ -311,34 +308,63 @@ function Invoke-DbaDbDataMasking {
$identityColumn = $dbTable.Columns | Where-Object Identity | Select-Object -ExpandProperty Name
}

# Check if the index for the identity column is already present
$maskingIndexName = "NIX__$($dbTable.Schema)_$($dbTable.Name)_Masking"
try {
if ($dbTable.Indexes.Name -contains $maskingIndexName) {
Write-Message -Level Verbose -Message "Masking index already exists in table [$($dbTable.Schema)].[$($dbTable.Name)]. Dropping it..."
$dbTable.Indexes[$($maskingIndexName)].Drop()
}
} catch {
Stop-Function -Message "Could not remove identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue
}

# Create the index for the identity column
try {
Write-Message -Level Verbose -Message "Adding index on identity column [$($identityColumn)] in table [$($dbTable.Schema)].[$($dbTable.Name)]"

$query = "CREATE NONCLUSTERED INDEX NIX_$($dbTable.Name)_Masking ON [$($dbTable.Schema)].[$($dbTable.Name)]([$($identityColumn)])"
$query = "CREATE NONCLUSTERED INDEX [$($maskingIndexName)] ON [$($dbTable.Schema)].[$($dbTable.Name)]([$($identityColumn)])"

Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database $db.Name -Query $query
} catch {
Stop-Function -Message "Could not add identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue
}


try {
if (-not (Test-Bound -ParameterName Query)) {
if (-not $tableobject.FilterQuery) {
# Get all the columns from the table
$columnString = "[" + (($dbTable.Columns | Where-Object DataType -in $supportedDataTypes | Select-Object Name -ExpandProperty Name) -join "],[") + "]"

# Add the identifier column
$columnString += ",[$($identityColumn)]"

# Put it all together
$query = "SELECT $($columnString) FROM [$($tableobject.Schema)].[$($tableobject.Name)]"
} else {
# Get the query from the table objects
$query = ($tableobject.FilterQuery).ToLower()

# Check if the query already contains the identifier column
if (-not ($query | Select-String -Pattern $identityColumn)) {
# Split up the query from the first "from"
$queryParts = $query -split "from", 2

# Put it all together again with the identifier
$query = "$($queryParts[0].Trim()), $($identityColumn) FROM $($queryParts[1].Trim())"
}
}
[array]$data = $db.Query($query)

# Get the data
[array]$data = $db.Query($query)
} catch {
Stop-Function -Message "Failure retrieving the data from table $($tableobject.Name)" -Target $Database -ErrorRecord $_ -Continue
Stop-Function -Message "Failure retrieving the data from table [$($tableobject.Schema)].[$($tableobject.Name)]" -Target $Database -ErrorRecord $_ -Continue
}

# Check if the table contains unique indexes
if ($tableobject.HasUniqueIndex) {

# Loop through the rows and generate a unique value for each row
Write-Message -Level Verbose -Message "Generating unique values for $($tableobject.Name)"
Write-Message -Level Verbose -Message "Generating unique values for [$($tableobject.Schema)].[$($tableobject.Name)]"

for ($i = 0; $i -lt $data.Count; $i++) {

Expand Down Expand Up @@ -936,16 +962,21 @@ function Invoke-DbaDbDataMasking {
}
}

# Clean up the masking index
try {
Write-Message -Level Verbose -Message "Removing index on identity column [$($identityColumn)] in table [$($dbTable.Schema)].[$($dbTable.Name)]"
# Refresh the indexes to make sure to have the latest list
$dbTable.Indexes.Refresh()

$query = "DROP INDEX [NIX_$($dbTable.Name)_Masking] ON [$($dbTable.Schema)].[$($dbTable.Name)]"

Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $query
# Check if the index is there
if ($dbTable.Indexes.Name -contains $maskingIndexName) {
Write-Message -Level verbose -Message "Removing identity index from table [$($dbTable.Schema)].[$($dbTable.Name)]"
$dbTable.Indexes[$($maskingIndexName)].Drop()
}
} catch {
Stop-Function -Message "Could not remove identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue
Stop-Function -Message "Could not remove identity index from table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue
}

# Clean up the identity column
if ($cleanupIdentityColumn) {
try {
Write-Message -Level Verbose -Message "Removing identity column [$($identityColumn)] from table [$($dbTable.Schema)].[$($dbTable.Name)]"
Expand All @@ -959,6 +990,7 @@ function Invoke-DbaDbDataMasking {
}
}

# Return the masking results
try {
[pscustomobject]@{
ComputerName = $db.Parent.ComputerName
Expand Down
4 changes: 2 additions & 2 deletions functions/New-DbaDbMaskingConfig.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ function New-DbaDbMaskingConfig {

# Loop through the tables
foreach ($tableobject in $tablecollection) {
Write-Message -Message "Processing table $($tableobject.Name)" -Level Verbose
Write-Message -Message "Processing table [$($tableobject.Schema)].[$($tableobject.Name)]" -Level Verbose

$hasUniqueIndex = $false

Expand Down Expand Up @@ -580,6 +580,7 @@ function New-DbaDbMaskingConfig {
Schema = $tableobject.Schema
Columns = $columns
HasUniqueIndex = $hasUniqueIndex
FilterQuery = $null
}
} else {
Write-Message -Message "No columns match for masking in table $($tableobject.Name)" -Level Verbose
Expand All @@ -603,7 +604,6 @@ function New-DbaDbMaskingConfig {
try {
$filenamepart = $server.Name.Replace('\', '$').Replace('TCP:', '').Replace(',', '.')
$temppath = Join-Path -Path $Path -ChildPath "$($filenamepart).$($db.Name).DataMaskingConfig.json"
#$temppath = "$Path\$($filenamepart).$($db.Name).DataMaskingConfig.json"

if (-not $script:isWindows) {
$temppath = $temppath.Replace("\", "/")
Expand Down
20 changes: 1 addition & 19 deletions functions/Test-DbaDbDataMaskingConfig.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,15 +280,6 @@ function Test-DbaDbDataMaskingConfig {
Error = "The category is not valid with data type $($column.ColumnType)"
}
}

<# if (-not $null -eq $column.Action.SubCategory -and $column.Action.SubCategory -notin $allo) {
[PSCustomObject]@{
Table = $table.Name
Column = $column.Name
Value = $column.Action.Category
Error = "The action subcategory '$($column.Action.SubCategory)' is not allowed"
}
} #>
}

# Number checks
Expand All @@ -304,15 +295,6 @@ function Test-DbaDbDataMaskingConfig {
Error = "The action does not contain all the required properties. Please check the action "
}
}

<# if ($compareResult.SideIndicator -contains "=>") {
[PSCustomObject]@{
Table = $table.Name
Column = $column.Name
Value = ($compareResult | Where-Object SideIndicator -eq "=>").InputObject -join ","
Error = "The action contains a property that is not in the required properties. Please check the column"
}
} #>
}

if ($column.ColumnType -notin $allowedNumberTypes) {
Expand All @@ -324,7 +306,7 @@ function Test-DbaDbDataMaskingConfig {
}
}
}
}# End column action
} # End column action
} # End for each column
} # End for each table
}
Expand Down
2 changes: 1 addition & 1 deletion tests/Invoke-DbaDbDataMasking.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Write-Host -Object "Running $PSCommandPath" -ForegroundColor Cyan
Describe "$CommandName Unit Tests" -Tag 'UnitTests' {
Context "Validate parameters" {
[object[]]$params = (Get-Command $CommandName).Parameters.Keys | Where-Object { $_ -notin ('whatif', 'confirm') }
[object[]]$knownParameters = 'SqlInstance', 'SqlCredential', 'Database', 'FilePath', 'Locale', 'CharacterString', 'Table', 'Column', 'ExcludeTable', 'ExcludeColumn', 'Query', 'MaxValue', 'ModulusFactor', 'ExactLength', 'ConnectionTimeout', 'CommandTimeout', 'BatchSize', 'DictionaryFilePath', 'DictionaryExportPath', 'EnableException'
[object[]]$knownParameters = 'SqlInstance', 'SqlCredential', 'Database', 'FilePath', 'Locale', 'CharacterString', 'Table', 'Column', 'ExcludeTable', 'ExcludeColumn', 'MaxValue', 'ModulusFactor', 'ExactLength', 'ConnectionTimeout', 'CommandTimeout', 'BatchSize', 'DictionaryFilePath', 'DictionaryExportPath', 'EnableException'
$knownParameters += [System.Management.Automation.PSCmdlet]::CommonParameters
It "Should only contain our specific parameters" {
(@(Compare-Object -ReferenceObject ($knownParameters | Where-Object { $_ }) -DifferenceObject $params).Count ) | Should Be 0
Expand Down

0 comments on commit 0964968

Please sign in to comment.