From 2d5ac17f5967e44bd854546b21005ce61a736c9f Mon Sep 17 00:00:00 2001 From: Chrissy LeMaire Date: Wed, 7 Oct 2020 16:41:28 +0200 Subject: [PATCH] 1.0.124 not in gallery yet, gallery is down --- allcommands.ps1 | 891 +++++++++++++++++++++++++++++++--------- bin/dbatools-index.json | 32 +- dbatools.psd1 | 2 +- en-us/dbatools-help.xml | 18 +- 4 files changed, 733 insertions(+), 210 deletions(-) diff --git a/allcommands.ps1 b/allcommands.ps1 index 1000502ab4..e38964452b 100644 --- a/allcommands.ps1 +++ b/allcommands.ps1 @@ -33359,7 +33359,22 @@ function Get-DbaRandomizedValue { } elseif ($randSubType -in 'string', 'string2') { $script:faker.Random.String2([int]$Min, [int]$Max, $CharacterString) } elseif ($randSubType -eq 'shuffle') { - $script:faker.Random.Shuffle($Value) + $commaIndex = $value.IndexOf(",") + $dotIndex = $value.IndexOf(".") + + $Value = (($Value -replace ',', '') -replace '\.', '') + + $newValue = ($script:faker.Random.Shuffle($Value) -join '') + + if ($commaIndex -ne -1) { + $newValue = $newValue.Insert($commaIndex, ',') + } + + if ($dotIndex -ne -1) { + $newValue = $newValue.Insert($dotIndex, '.') + } + + $newValue } else { $script:faker.Random.$RandomizerSubType() } @@ -42835,11 +42850,12 @@ function Invoke-DbaDbDataMasking { [string[]]$ExcludeTable, [string[]]$ExcludeColumn, [int]$MaxValue, - [int]$ModulusFactor = 10, + [int]$ModulusFactor, [switch]$ExactLength, - [int]$ConnectionTimeout = 0, - [int]$CommandTimeout = 300, - [int]$BatchSize = 1000, + [int]$ConnectionTimeout, + [int]$CommandTimeout, + [int]$BatchSize, + [int]$Retry, [string[]]$DictionaryFilePath, [string]$DictionaryExportPath, [switch]$EnableException @@ -42867,6 +42883,32 @@ function Invoke-DbaDbDataMasking { $supportedFakerSubTypes += "Date" + # Set defaults + if (-not $ModulusFactor) { + $ModulusFactor = 10 + Write-Message -Level Verbose -Message "Modulus factor set to $ModulusFactor" + } + + if (-not $ConnectionTimeout) { + $ConnectionTimeout = 0 + Write-Message -Level Verbose -Message "Connection time-out set to $ConnectionTimeout" + } + + if (-not $CommandTimeout) { + $CommandTimeout = 300 + Write-Message -Level Verbose -Message "Command time-out set to $CommandTimeout" + } + + if (-not $BatchSize) { + $BatchSize = 1000 + Write-Message -Level Verbose -Message "Batch size set to $BatchSize" + } + + if (-not $Retry) { + $Retry = 1000 + Write-Message -Level Verbose -Message "Retry count set to $Retry" + } + # Import the dictionary files if ($DictionaryFilePath.Count -ge 1) { $dictionary = @{ } @@ -42967,10 +43009,12 @@ function Invoke-DbaDbDataMasking { $db = $server.Databases[$($dbName)] - $stepcounter = $nullmod = 0 + #$stepcounter = $nullmod = 0 + $nullmod = 0 + #region for each table foreach ($tableobject in $tables.Tables) { - $uniqueValues = @() + $uniqueDataTableName = $null $uniqueValueColumns = @() $stringBuilder = [System.Text.StringBuilder]'' @@ -42988,7 +43032,7 @@ function Invoke-DbaDbDataMasking { [bool]$cleanupIdentityColumn = $false # Make sure there is an identity column present to speed things up - if (-not ($dbTable.Columns | Where-Object Identity -eq $true)) { + if (-not ($dbTable.Columns | Where-Object { $_.Identity -eq $true })) { Write-Message -Level Verbose -Message "Adding identity column to table [$($dbTable.Schema)].[$($dbTable.Name)]" $query = "ALTER TABLE [$($dbTable.Schema)].[$($dbTable.Name)] ADD MaskingID BIGINT IDENTITY(1, 1) NOT NULL;" @@ -43000,7 +43044,7 @@ function Invoke-DbaDbDataMasking { $dbTable.Columns.Refresh() } else { - $identityColumn = $dbTable.Columns | Where-Object Identity | Select-Object -ExpandProperty Name + $identityColumn = $dbTable.Columns | Where-Object { $_.Identity } | Select-Object -ExpandProperty Name } # Check if the index for the identity column is already present @@ -43028,7 +43072,7 @@ function Invoke-DbaDbDataMasking { try { if (-not $tableobject.FilterQuery) { # Get all the columns from the table - $columnString = "[" + (($dbTable.Columns | Where-Object DataType -in $supportedDataTypes | Select-Object Name -ExpandProperty Name) -join "],[") + "]" + $columnString = "[" + (($dbTable.Columns | Where-Object { $_.DataType -in $supportedDataTypes } | Select-Object Name -ExpandProperty Name) -join "],[") + "]" # Add the identifier column $columnString += ",[$($identityColumn)]" @@ -43055,28 +43099,195 @@ function Invoke-DbaDbDataMasking { Stop-Function -Message "Failure retrieving the data from table [$($tableobject.Schema)].[$($tableobject.Name)]" -Target $Database -ErrorRecord $_ -Continue } + #region unique indexes # Check if the table contains unique indexes if ($tableobject.HasUniqueIndex) { # Loop through the rows and generate a unique value for each row Write-Message -Level Verbose -Message "Generating unique values for [$($tableobject.Schema)].[$($tableobject.Name)]" - for ($i = 0; $i -lt $data.Count; $i++) { + $params = @{ + SqlInstance = $server + SqlCredential = $SqlCredential + Database = $db.name + Schema = $tableobject.Schema + Table = $tableobject.Name + } - $rowValue = New-Object PSCustomObject + $indexToTable = Convert-DbaIndexToTable @params - # Loop through each of the unique indexes - foreach ($index in ($db.Tables[$($tableobject.Name)].Indexes | Where-Object IsUnique -eq $true )) { + # compare the index columns to the column in the json table object + $compareParams = @{ + ReferenceObject = $indexToTable.Columns + DifferenceObject = $tableobject.Columns.Name + IncludeEqual = $true + } + $maskingColumnIndexCount = (Compare-Object @compareParams | Where-Object { $_.SideIndicator -eq "==" }).Count - # Loop through the index columns - foreach ($indexColumn in $index.IndexedColumns) { + # Check if there is any need to generate unique values + if ($maskingColumnIndexCount -ge 1) { - if (-not $dbTable.Columns[$indexColumn.Name].Identity) { + # Check if the temporary table already exists + $server.Databases['tempdb'].Tables.Refresh() + $uniqueDataTableName = $indexToTable.TempTableName - # Get the column mask info - $columnMaskInfo = $tableobject.Columns | Where-Object Name -eq $indexColumn.Name + if ($server.Databases['tempdb'].Tables.Name -contains $indexToTable.TempTableName) { + Write-Message -Level Verbose -Message "Table '$($indexToTable.TempTableName)' already exists. Dropping it.." + try { + $query = "DROP TABLE $($indexToTable.TempTableName)" + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $query + } catch { + Stop-Function -Message "Could not drop temporary table" + } + } - if ($columnMaskInfo) { + # Create the temporary table + try { + Write-Message -Level Verbose -Message "Creating temporary table '$($indexToTable.TempTableName)'" + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $indexToTable.CreateStatement + } catch { + Stop-Function -Message "Could not create temporary table #[$($tableobject.Schema)].[$($tableobject.Name)]" + } + + # Create the unique index table + try { + Write-Message -Level Verbose -Message "Creating the unique index for temporary table '$($indexToTable.TempTableName)'" + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $indexToTable.UniqueIndexStatement + } catch { + Stop-Function -Message "Could not create temporary table #[$($tableobject.Schema)].[$($tableobject.Name)]" + } + + # Create a unique row + $retryCount = 0 + for ($i = 0; $i -lt $data.Count; $i++) { + $insertQuery = "INSERT INTO [$($indexToTable.TempTableName)]([$($indexToTable.Columns -join '],[')]) VALUES(" + $insertFailed = $false + $insertValues = @() + + #[array]$paramArray = @() + + foreach ($indexColumn in $indexToTable.Columns) { + $columnMaskInfo = $tableobject.Columns | Where-Object { $_.Name -eq $indexColumn } + + if ($indexColumn -eq "RowNr") { + $newValue = $i + 1 + } elseif ($columnMaskInfo) { + # make sure min is good + if ($columnMaskInfo.MinValue) { + $min = $columnMaskInfo.MinValue + } else { + if ($columnMaskInfo.CharacterString) { + $min = 1 + } else { + $min = 0 + } + } + + # make sure max is good + if ($MaxValue) { + if ($columnMaskInfo.MaxValue -le $MaxValue) { + $max = $columnMaskInfo.MaxValue + } else { + $max = $MaxValue + } + } else { + $max = $columnMaskInfo.MaxValue + } + + if (-not $columnMaskInfo.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { + $max = 10 + } + + if ((-not $columnMaskInfo.MinValue -or -not $columnMaskInfo.MaxValue) -and ($columnMaskInfo.ColumnType -match 'date')) { + if (-not $columnMaskInfo.MinValue) { + $min = (Get-Date).AddDays(-365) + } + if (-not $columnMaskInfo.MaxValue) { + $max = (Get-Date).AddDays(365) + } + } + + if ($columnMaskInfo.CharacterString) { + $charstring = $columnMaskInfo.CharacterString + } else { + $charstring = $CharacterString + } + + # Generate a new value + $newValue = $null + + $newValueParams = $null + + try { + $newValueParams = $null + if (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { + $newValueParams = @{ + DataType = $columnMaskInfo.SubType + Min = $columnMaskInfo.MinValue + Max = $columnMaskInfo.MaxValue + Locale = $Locale + } + } else { + $newValueParams = @{ + RandomizerType = $columnMaskInfo.MaskingType + RandomizerSubtype = $columnMaskInfo.SubType + Min = $min + Max = $max + CharacterString = $charstring + Format = $columnMaskInfo.Format + Separator = $columnMaskInfo.Separator + Locale = $Locale + } + } + + $newValue = Get-DbaRandomizedValue @newValueParams + } catch { + Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ + } + } else { + $newValue = $null + } + + if ($columnMaskInfo) { + $insertValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnMaskInfo.ColumnType -Nullable:$columnMaskInfo.Nullable + + $insertValues += $insertValue.NewValue + } elseif ($indexColumn -eq "RowNr") { + $insertValues += $newValue + } else { + $insertValues += "NULL" + } + + $uniqueValueColumns += $columnMaskInfo.Name + } + + # Join all the values to the insert query + $insertQuery += "$($insertValues -join ','));" + + # Try inserting the value + try { + $server.Databases['tempdb'].Query($insertQuery) + $insertFailed = $false + } catch { + Write-PSFMessage -Level Verbose -Message "Could not insert value" + $insertFailed = $true + } + + # Try to insert the value as long it's failed + while ($insertFailed) { + if ($retryCount -eq $Retry) { + Stop-Function -Message "Could not create a unique row after $retryCount tries. Stopping..." + return + } + + $insertQuery = "INSERT INTO [$($indexToTable.TempTableName)]([$($indexToTable.Columns -join '],[')]) VALUES(" + + foreach ($indexColumn in $indexToTable.Columns) { + $columnMaskInfo = $tableobject.Columns | Where-Object { $_.Name -eq $indexColumn } + + if ($indexColumn -eq "RowNr") { + $newValue = $i + 1 + } elseif ($columnMaskInfo) { # make sure min is good if ($columnMaskInfo.MinValue) { $min = $columnMaskInfo.MinValue @@ -43112,14 +43323,19 @@ function Invoke-DbaDbDataMasking { } } - if ($columnobject.CharacterString) { + if ($columnMaskInfo.CharacterString) { $charstring = $columnMaskInfo.CharacterString } else { $charstring = $CharacterString } # Generate a new value + $newValue = $null + + $newValueParams = $null + try { + $newValueParams = $null if (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { $newValueParams = @{ DataType = $columnMaskInfo.SubType @@ -43127,8 +43343,6 @@ function Invoke-DbaDbDataMasking { Max = $columnMaskInfo.MaxValue Locale = $Locale } - - $newValue = Get-DbaRandomizedValue @newValueParams } else { $newValueParams = @{ RandomizerType = $columnMaskInfo.MaskingType @@ -43140,82 +43354,61 @@ function Invoke-DbaDbDataMasking { Separator = $columnMaskInfo.Separator Locale = $Locale } - - $newValue = Get-DbaRandomizedValue @newValueParams } + $newValue = Get-DbaRandomizedValue @newValueParams } catch { Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ } - - # Check if the value is already present as a property - if (($rowValue | Get-Member -MemberType NoteProperty).Name -notcontains $indexColumn.Name) { - $rowValue | Add-Member -Name $indexColumn.Name -Type NoteProperty -Value $newValue - } + } else { + $newValue = $null } - # To be sure the values are unique, loop as long as long as needed to generate a unique value - while (($uniqueValues | Select-Object -Property ($rowValue | Get-Member -MemberType NoteProperty | Select-Object -ExpandProperty Name)) -match $rowValue) { - - $rowValue = New-Object PSCustomObject - - # Loop through the index columns - foreach ($indexColumn in $index.IndexedColumns) { - - # Get the column mask info - $columnMaskInfo = $tableobject.Columns | Where-Object Name -eq $indexColumn.Name - - if ($columnMaskInfo) { - # Generate a new value - try { - if (-not $columnMaskInfo.SubType -and $columnMaskInfo.ColumnType -in $supportedDataTypes) { - $newValueParams = @{ - DataType = $columnMaskInfo.SubType - Min = $columnMaskInfo.MinValue - Max = $columnMaskInfo.MaxValue - Locale = $Locale - } - - $newValue = Get-DbaRandomizedValue @newValueParams - } else { - $newValueParams = @{ - RandomizerType = $columnMaskInfo.MaskingType - RandomizerSubtype = $columnMaskInfo.SubType - Min = $min - Max = $max - CharacterString = $charstring - Format = $columnMaskInfo.Format - Separator = $columnMaskInfo.Separator - Locale = $Locale - } - $newValue = Get-DbaRandomizedValue @newValueParams - } - - } catch { - Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ - } + if ($columnMaskInfo) { + $insertValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnMaskInfo.ColumnType -Nullable:$columnMaskInfo.Nullable - # Check if the value is already present as a property - if (($rowValue | Get-Member -MemberType NoteProperty).Name -notcontains $indexColumn.Name) { - $rowValue | Add-Member -Name $indexColumn.Name -Type NoteProperty -Value $newValue - } - } - } + $insertValues += $insertValue.NewValue + } elseif ($indexColumn -eq "RowNr") { + $insertValues += $newValue + } else { + $insertValues += "NULL" } } - # Add the row value to the array - $uniqueValues += $rowValue + + # Join all the values to the insert query + $insertQuery += "$($insertValues -join ','));" + + # Try inserting the value + try { + $server.Databases['tempdb'].Query($insertQuery) + $insertFailed = $false + } catch { + Write-PSFMessage -Level Verbose -Message "Could not insert value" + $insertQuery + $insertFailed = $true + $retryCount++ + } } } + + try { + Write-Message -Level Verbose -Message "Creating masking index for [$($indexToTable.TempTableName)]" + $query = "CREATE NONCLUSTERED INDEX [NIX_$($indexToTable.TempTableName)_MaskID] ON [$($indexToTable.TempTableName)]([RowNr])" + $server.Databases['tempdb'].Query($query) + } catch { + Stop-Function -Message "Could not add masking index for [$($indexToTable.TempTableName)]" -ErrorRecord $_ + } + } else { + Write-PSFMessage -Level Verbose -Message "Table [$($tableobject.Schema)].[$($tableobject.Name)] does not contain any masking index columns to process" } } - $uniqueValueColumns = $uniqueValueColumns | Select-Object -Unique + #endregion unique indexes $tablecolumns = $tableobject.Columns if ($Column) { - $tablecolumns = $tablecolumns | Where-Object Name -in $Column + $tablecolumns = $tablecolumns | Where-Object { $_.Name -in $Column } } if ($ExcludeColumn) { @@ -43223,11 +43416,11 @@ function Invoke-DbaDbDataMasking { Stop-Function -Message "Column present in -ExcludeColumn cannot be excluded because it's part of an unique index" -Target $ExcludeColumn -Continue } - $tablecolumns = $tablecolumns | Where-Object Name -notin $ExcludeColumn + $tablecolumns = $tablecolumns | Where-Object { $_.Name -notin $ExcludeColumn } } if (-not $tablecolumns) { - Write-Message -Level Verbose "No columns to process in $($dbName).$($tableobject.Schema).$($tableobject.Name), moving on" + Write-Message -Level Verbose "No columns to process in [$($dbName)].[$($tableobject.Schema)].[$($tableobject.Name)], moving on" continue } @@ -43235,12 +43428,13 @@ function Invoke-DbaDbDataMasking { $elapsed = [System.Diagnostics.Stopwatch]::StartNew() $totalBatches = [System.Math]::Ceiling($data.Count / $BatchSize) - $rowNumber = $stepcounter = $batchRowCounter = $batchCounter = 0 + #$rowNumber = $stepcounter = $batchRowCounter = $batchCounter = 0 + $rowNumber = $batchRowCounter = $batchCounter = 0 $columnsWithActions = @() - $columnsWithActions += $tableobject.Columns | Where-Object Action -ne $null + $columnsWithActions += $tableobject.Columns | Where-Object { $null -ne $_.Action } - # Go through the composites + # Go through the actions if ($columnsWithActions.Count -ge 1) { foreach ($columnObject in $columnsWithActions) { [bool]$validAction = $true @@ -43321,11 +43515,22 @@ function Invoke-DbaDbDataMasking { # Loop through each of the rows and change them foreach ($columnobject in $tablecolumns) { - $batchCounter = 1 + $batchCounter = 0 - # Check if column is does not contain an action - if ($columnobject.Name -notin $columnsWithActions.Name) { + if ($columnobject.StaticValue) { + $newValue = $columnobject.StaticValue + + if ($null -eq $newValue -and -not $columnobject.Nullable) { + Write-PSFMessage -Message "Column '$($columnobject.Name)' static value cannot null when column is set not to be nullable." + } else { + $convertedValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnobject.ColumnType -Nullable:$columnobject.Nullable + $null = $stringBuilder.AppendLine("UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET [$($columnObject.Name)] = $($convertedValue.NewValue)") + $batchRowCounter++ + } + } elseif ($columnobject.Name -notin $columnsWithActions.Name) { + # Column does not have an action foreach ($row in $data) { + $rowNumber++ if ((($batchRowCounter++) % 100) -eq 0) { $progressParams = @{ @@ -43354,19 +43559,22 @@ function Invoke-DbaDbDataMasking { Stop-Function -Message "Unsupported masking sub type '$($columnobject.SubType)' for column $($columnobject.Name)" -Target $columnobject -Continue } - if ($columnobject.KeepNull -and (($row.($columnobject.Name)).GetType().Name -eq 'DBNull')) { + if ($columnobject.MaskingType -eq 'Static') { + $newValue = $columnobject.StaticValue + } elseif ($columnobject.KeepNull -and (($row.($columnobject.Name)).GetType().Name -eq 'DBNull')) { $newValue = $null } elseif (-not $columnobject.KeepNull -and $columnobject.Nullable -and (($nullmod++) % $ModulusFactor -eq 0)) { $newValue = $null } elseif ($tableobject.HasUniqueIndex -and $columnobject.Name -in $uniqueValueColumns) { + $query = "SELECT $($columnobject.Name) FROM $($uniqueDataTableName) Where [RowNr] = $rowNumber" + $uniqueData = Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database tempdb -Query $query - if ($uniqueValues.Count -lt 1) { - Stop-Function -Message "Could not find any unique values in dictionary" -Target $tableobject + if ($null -eq $uniqueData) { + Stop-Function -Message "Could not find any unique values" -Target $tableobject return } - $newValue = $uniqueValues[$rowNumber].$($columnobject.Name) - + $newValue = $uniqueData.$($columnobject.Name) } elseif ($columnobject.Deterministic -and $dictionary.ContainsKey($row.$($columnobject.Name) )) { $newValue = $dictionary.Item($row.$($columnobject.Name)) } else { @@ -43423,32 +43631,13 @@ function Invoke-DbaDbDataMasking { Value = ($row.$($columnobject.Name)) Locale = $Locale } - - $newValue = Get-DbaRandomizedValue @newValueParams - - $newValue = ($newValue -join '') } elseif ($columnobject.ColumnType -in 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real') { - $valueString = ($row.$($columnobject.Name)).ToString() - - $commaIndex = $valueString.IndexOf(",") - $dotIndex = $valueString.IndexOf(".") - $newValueParams = @{ RandomizerType = "Random" RandomizerSubtype = "Shuffle" - Value = (($valueString -replace ',', '') -replace '\.', '') -join '' + Value = ($row.$($columnobject.Name)) Locale = $Locale } - - $newValue = Get-DbaRandomizedValue @newValueParams - - if ($commaIndex -ne -1) { - $newValue = $newValue.Insert($commaIndex, ',') - } - - if ($dotIndex -ne -1) { - $newValue = $newValue.Insert($dotIndex, '.') - } } } elseif (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { $newValueParams = @{ @@ -43459,8 +43648,6 @@ function Invoke-DbaDbDataMasking { Format = $columnobject.Format Locale = $Locale } - - $newValue = Get-DbaRandomizedValue @newValueParams } else { $newValueParams = @{ RandomizerType = $columnobject.MaskingType @@ -43472,55 +43659,27 @@ function Invoke-DbaDbDataMasking { Separator = $columnobject.Separator Locale = $Locale } - - $newValue = Get-DbaRandomizedValue @newValueParams } + + $newValue = Get-DbaRandomizedValue @newValueParams } catch { Stop-Function -Message "Failure" -Target $columnobject -Continue -ErrorRecord $_ } } - if ($null -eq $newValue -and $columnobject.Nullable -eq $true) { - $updates += "[$($columnobject.Name)] = NULL" - } elseif ($columnobject.ColumnType -in 'bit', 'bool') { - if ($columnValue) { - $updates += "[$($columnobject.Name)] = 1" - } else { - $updates += "[$($columnobject.Name)] = 0" - } - } elseif ($columnobject.ColumnType -like '*int*' -or $columnobject.ColumnType -in 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real') { - $updates += "[$($columnobject.Name)] = $newValue" - } elseif ($columnobject.ColumnType -in 'uniqueidentifier') { - $updates += "[$($columnobject.Name)] = '$newValue'" - } elseif ($columnobject.ColumnType -eq 'datetime') { - $newValue = ([datetime]$newValue).Tostring("yyyy-MM-dd HH:mm:ss.fff") - $updates += "[$($columnobject.Name)] = '$newValue'" - } elseif ($columnobject.ColumnType -eq 'datetime2') { - $newValue = ([datetime]$newValue).Tostring("yyyy-MM-dd HH:mm:ss.fffffff") - $updates += "[$($columnobject.Name)] = '$newValue'" - } elseif ($columnobject.ColumnType -like 'date') { - $newValue = ([datetime]$newValue).Tostring("yyyy-MM-dd") - $updates += "[$($columnobject.Name)] = '$newValue'" - } elseif ($columnobject.ColumnType -like '*date*') { - $newValue = ([datetime]$newValue).Tostring("yyyy-MM-dd HH:mm:ss") - $updates += "[$($columnobject.Name)] = '$newValue'" - } elseif ($columnobject.ColumnType -like 'time') { - $newValue = ([datetime]$newValue).Tostring("HH:mm:ss.fffffff") - $updates += "[$($columnobject.Name)] = '$newValue'" - } elseif ($columnobject.ColumnType -eq 'xml') { - # nothing, unsure how i'll handle this - } else { - $newValue = ($newValue).Tostring().Replace("'", "''") - $updates += "[$($columnobject.Name)] = '$newValue'" - } + $convertedValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnobject.ColumnType -Nullable:$columnobject.Nullable + $updates += "[$($columnobject.Name)] = $($convertedValue.NewValue)" + # Check if this value is determinisic if ($columnobject.Deterministic -and -not $dictionary.ContainsKey($row.$($columnobject.Name) )) { $dictionary.Add($row.$($columnobject.Name), $newValue) } + # Setup the query + $updateQuery = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE [$($identityColumn)] = $($row.$($identityColumn)); " + $null = $stringBuilder.AppendLine($updateQuery) - $null = $stringBuilder.AppendLine("UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE [$($identityColumn)] = $($row.$($identityColumn)); ") } if ($batchRowCounter -eq $BatchSize) { @@ -43548,9 +43707,6 @@ function Invoke-DbaDbDataMasking { $null = $stringBuilder.Clear() $batchRowCounter = 0 } - - # Increase the row number - $rowNumber++ } if ($stringBuilder.Length -ge 1) { @@ -43578,7 +43734,7 @@ function Invoke-DbaDbDataMasking { $null = $stringBuilder.Clear() $columnsWithComposites = @() - $columnsWithComposites += $tableobject.Columns | Where-Object Composite -ne $null + $columnsWithComposites += $tableobject.Columns | Where-Object { $null -ne $_.Composite } # Check for both special actions if (($columnsWithComposites.Count -ge 1) -and ($columnsWithActions.Count -ge 1)) { @@ -43704,9 +43860,18 @@ function Invoke-DbaDbDataMasking { } } - # Empty the unique values array - $uniqueValues = $null + # Cleanup + if ($uniqueDataTableName) { + Write-Message -Message "Cleaning up unique temporary table '$uniqueDataTableName'" -Level verbose + $query = "DROP TABLE [$($uniqueDataTableName)];" + try { + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $query + } catch { + Stop-Function -Message "Could not clean up unique values table '$uniqueDataTableName'" -Target $uniqueDataTableName -ErrorRecord $_ + } + } } + #endregion for each table # Export the dictionary when needed if ($DictionaryExportPath) { @@ -53341,6 +53506,7 @@ function New-DbaDbMaskingConfig { KeepNull = $true Composite = $null Action = $null + StaticValue = $null } } else { $type = "Random" @@ -53407,6 +53573,7 @@ function New-DbaDbMaskingConfig { KeepNull = $true Composite = $null Action = $null + StaticValue = $null } } } @@ -67542,13 +67709,21 @@ function Set-DbaTempDbConfig { $logfile = Get-DbaDbFile -SqlInstance $server -Database tempdb | Where-Object Type -eq 1 | Select-Object LogicalName, PhysicalName, @{L = "SizeMb"; E = { $_.Size.Megabyte } } - if ($LogFileSize) { + if ($LogPath -or $LogFileSize) { $Filename = Split-Path $logfile.PhysicalName -Leaf $LogicalName = $logfile.LogicalName - $NewPath = "$LogPath\$Filename" + + if ($LogPath) { + $NewPath = "$LogPath\$Filename" + } else { + $NewPath = $logfile.PhysicalName + } + + if (-not($LogFileSize)) { + $LogFileSize = $logfile.SizeMb + } + $sql += "ALTER DATABASE tempdb MODIFY FILE(name=$LogicalName,filename='$NewPath',size=$LogFileSize MB,filegrowth=$LogFileGrowth);" - } else { - $LogFileSize = $logfile.SizeMb } Write-Message -Message "SQL Statement to resize tempdb." -Level Verbose @@ -71183,7 +71358,7 @@ function Test-DbaDbDataMaskingConfig { $randomizerTypes = Get-DbaRandomizedType $requiredColumnProperties = @('Action', 'CharacterString', 'ColumnType', 'Composite', 'Deterministic', 'Format', 'MaskingType', 'MaxValue', 'MinValue', 'Name', 'Nullable', 'KeepNull', 'SubType') - $allowedColumnProperties = @('Action', 'CharacterString', 'ColumnType', 'Composite', 'Deterministic', 'Format', 'MaskingType', 'MaxValue', 'MinValue', 'Name', 'Nullable', 'KeepNull', 'Separator', 'SubType') + $allowedColumnProperties = @('Action', 'CharacterString', 'ColumnType', 'Composite', 'Deterministic', 'Format', 'MaskingType', 'MaxValue', 'MinValue', 'Name', 'Nullable', 'KeepNull', 'Separator', 'SubType', 'StaticValue') $allowedActionCategories = @('datetime', 'number', 'column') $allowedActionSubCategories = @('year', 'quarter', 'month', 'dayofyear', 'day', 'week', 'weekday', 'hour', 'minute', 'second', 'millisecond', 'microsecond', 'nanosecond') @@ -72227,46 +72402,102 @@ function Test-DbaDiskSpeed { [PSCredential]$SqlCredential, [object[]]$Database, [object[]]$ExcludeDatabase, + [ValidateSet('Database', 'Disk', 'File')] + [string]$AggregateBy = 'File', [switch]$EnableException ) begin { - $sql = "SELECT SERVERPROPERTY('MachineName') AS ComputerName, - ISNULL(SERVERPROPERTY('InstanceName'), 'MSSQLSERVER') AS InstanceName, - SERVERPROPERTY('ServerName') AS SqlInstance, db_name(a.database_id) AS [Database] - , CAST(((a.size_on_disk_bytes/1024)/1024.0)/1024 AS DECIMAL(10,2)) AS [SizeGB] - , RIGHT(b.physical_name, CHARINDEX('\', REVERSE(b.physical_name)) -1) AS [FileName] - , a.file_id AS [FileID] - , CASE WHEN a.file_id = 2 THEN 'Log' ELSE 'Data' END AS [FileType] - , UPPER(SUBSTRING(b.physical_name, 1, 2)) AS [DiskLocation] - , a.num_of_reads AS [Reads] - , CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END AS [AverageReadStall] - , CASE - WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END < 10 THEN 'Very Good' - WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END < 20 THEN 'OK' - WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END < 50 THEN 'Slow, Needs Attention' - WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END >= 50 THEN 'Serious I/O Bottleneck' - END AS [ReadPerformance] - , a.num_of_writes AS [Writes] - , CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/a.num_of_writes AS INT) END AS [AverageWriteStall] - , CASE - WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END < 10 THEN 'Very Good' - WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END < 20 THEN 'OK' - WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END < 50 THEN 'Slow, Needs Attention' - WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END >= 50 THEN 'Serious I/O Bottleneck' - END AS [WritePerformance] - FROM sys.dm_io_virtual_file_stats (NULL, NULL) a - JOIN sys.master_files b - ON a.file_id = b.file_id - AND a.database_id = b.database_id" + $sql = $null + + # Consolidating the common SQL to hopefully make the maintenance easier. The various scenarios are enabled by uncommenting specific lines at runtime. + $selectList = + "SELECT + SERVERPROPERTY('MachineName') AS ComputerName + , ISNULL(SERVERPROPERTY('InstanceName'), 'MSSQLSERVER') AS InstanceName + , SERVERPROPERTY('ServerName') AS SqlInstance + --DATABASE-SELECT, DB_NAME(a.database_id) AS [Database] + --FILE-ALL, CAST(((a.size_on_disk_bytes/1024)/1024.0)/1024 AS DECIMAL(10,2)) AS [SizeGB] + --FILE-WINDOWS, RIGHT(b.physical_name, CHARINDEX('\', REVERSE(b.physical_name)) -1) AS [FileName] + --FILE-LINUX, RIGHT(b.physical_name, CHARINDEX('/', REVERSE(b.physical_name)) -1) AS [FileName] + --FILE-ALL, a.file_id AS [FileID] + --FILE-ALL, CASE WHEN a.file_id = 2 THEN 'Log' ELSE 'Data' END AS [FileType] + --DATABASE-OR-DISK, a.DiskLocation AS DiskLocation + --FILE-WINDOWS, UPPER(SUBSTRING(b.physical_name, 1, 2)) AS DiskLocation + --FILE-LINUX, SUBSTRING(physical_name, 1, CHARINDEX('/', physical_name, CHARINDEX('/', physical_name) + 1) - 1) AS DiskLocation + , a.num_of_reads AS [Reads] + , CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END AS [AverageReadStall] + , CASE + WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END < 10 THEN 'Very Good' + WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END < 20 THEN 'OK' + WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END < 50 THEN 'Slow, Needs Attention' + WHEN CASE WHEN a.num_of_reads < 1 THEN NULL ELSE CAST(a.io_stall_read_ms/(a.num_of_reads) AS INT) END >= 50 THEN 'Serious I/O Bottleneck' + END AS [ReadPerformance] + , a.num_of_writes AS [Writes] + , CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/a.num_of_writes AS INT) END AS [AverageWriteStall] + , CASE + WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END < 10 THEN 'Very Good' + WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END < 20 THEN 'OK' + WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END < 50 THEN 'Slow, Needs Attention' + WHEN CASE WHEN a.num_of_writes < 1 THEN NULL ELSE CAST(a.io_stall_write_ms/(a.num_of_writes) AS INT) END >= 50 THEN 'Serious I/O Bottleneck' + END AS [WritePerformance] + , CASE + WHEN (a.num_of_reads = 0 AND a.num_of_writes = 0) THEN NULL + ELSE (a.io_stall/(a.num_of_reads + a.num_of_writes)) + END AS [Avg Overall Latency] + , CASE + WHEN a.num_of_reads = 0 THEN NULL + ELSE (a.num_of_bytes_read/a.num_of_reads) + END AS [Avg Bytes/Read] + , CASE + WHEN a.num_of_writes = 0 THEN NULL + ELSE (a.num_of_bytes_written/a.num_of_writes) + END AS [Avg Bytes/Write] + , CASE + WHEN (a.num_of_reads = 0 AND a.num_of_writes = 0) THEN NULL + ELSE ((a.num_of_bytes_read + a.num_of_bytes_written)/(a.num_of_reads + a.num_of_writes)) + END AS [Avg Bytes/Transfer]" + + if ($AggregateBy -eq 'File') { + $sql = "$selectList + FROM sys.dm_io_virtual_file_stats (NULL, NULL) a + JOIN sys.master_files b + ON a.file_id = b.file_id + AND a.database_id = b.database_id" + + } elseif ($AggregateBy -in ('Database', 'Disk')) { + $sql = "$selectList + FROM + ( + SELECT + --WINDOWS UPPER(SUBSTRING(b.physical_name, 1, 2)) AS DiskLocation + --LINUX SUBSTRING(physical_name, 1, CHARINDEX('/', physical_name, CHARINDEX('/', physical_name) + 1) - 1) AS DiskLocation + , SUM(a.num_of_reads) AS num_of_reads + , SUM(a.io_stall_read_ms) AS io_stall_read_ms + , SUM(a.num_of_writes) AS num_of_writes + , SUM(a.io_stall_write_ms) AS io_stall_write_ms + , SUM(a.num_of_bytes_read) AS num_of_bytes_read + , SUM(a.num_of_bytes_written) AS num_of_bytes_written + , SUM(a.io_stall) AS io_stall + --DATABASE-GROUPBY, a.database_id AS database_id + FROM sys.dm_io_virtual_file_stats (NULL, NULL) a + JOIN sys.master_files b + ON a.file_id = b.file_id + AND a.database_id = b.database_id + GROUP BY + --DATABASE-GROUPBYa.database_id, + --WINDOWS UPPER(SUBSTRING(b.physical_name, 1, 2)) + --LINUX SUBSTRING(physical_name, 1, CHARINDEX('/', physical_name, CHARINDEX('/', physical_name) + 1) - 1) + ) AS a" + } if ($Database -or $ExcludeDatabase) { - if ($database) { - $where = " where db_name(a.database_id) in ('$($Database -join "'")') " + if ($Database) { + $where = " where db_name(a.database_id) in ('$($Database -join "','")') " } if ($ExcludeDatabase) { - $where = " where db_name(a.database_id) not in ('$($ExcludeDatabase -join "'")') " + $where = " where db_name(a.database_id) not in ('$($ExcludeDatabase -join "','")') " } $sql += $where } @@ -72277,13 +72508,41 @@ function Test-DbaDiskSpeed { process { foreach ($instance in $SqlInstance) { + $sqlToRun = $sql + try { $server = Connect-SqlInstance -SqlInstance $instance -SqlCredential $SqlCredential -MinimumVersion 9 + + # At runtime uncomment the relevant pieces in the SQL + if ($AggregateBy -eq 'File') { + + $sqlToRun = $sqlToRun.replace("--DATABASE-SELECT", "").replace("--FILE-ALL", "") + + if ($server.HostPlatform -eq "Linux") { + $sqlToRun = $sqlToRun.replace("--FILE-LINUX", "") + } else { + $sqlToRun = $sqlToRun.replace("--FILE-WINDOWS", "") + } + + } elseif ($AggregateBy -in ('Database', 'Disk')) { + + $sqlToRun = $sqlToRun.replace("--DATABASE-OR-DISK", "") + + if ($server.HostPlatform -eq "Linux") { + $sqlToRun = $sqlToRun.replace("--LINUX", "") + } else { + $sqlToRun = $sqlToRun.replace("--WINDOWS", "") + } + + if ($AggregateBy -eq 'Database') { + $sqlToRun = $sqlToRun.replace("--DATABASE-SELECT", "").replace("--DATABASE-GROUPBY", "") + } + } } catch { Stop-Function -Message "Error occurred while establishing connection to $instance" -Category ConnectionError -ErrorRecord $_ -Target $instance -Continue } - Write-Message -Level Debug -Message "Executing $sql" - $server.Query("$sql") + Write-Message -Level Debug -Message "Executing $sqlToRun" + $server.Query("$sqlToRun") } } } @@ -76350,6 +76609,122 @@ function Convert-ByteToHexString { $outString } +#.ExternalHelp dbatools-Help.xml +function Convert-DbaIndexToTable { + [CmdLetBinding()] + + param( + [DbaInstanceParameter]$SqlInstance, + [PSCredential]$SqlCredential, + [string[]]$Database, + [string[]]$Schema, + [string[]]$Table, + [string[]]$Index, + [switch]$Unique, + [switch]$EnableException + ) + + begin { + # Some parameter checking + if (-not $SqlInstance) { + Stop-Function -Message "Please enter an instance" -Continue + } + + # Get the database + $db = Get-DbaDatabase -SqlInstance $SqlInstance -SqlCredential $SqlCredential -Database $Database + + # Stringbuilder to get store all the queries + $tableStatements = @() + } + + process { + if (Test-FunctionInterrupt) { return } + + # Filter all the schemas + if ($Schema) { + [array]$tables = $db.Tables | Where-Object { $_.Schema -in $Schema } + } else { + [array]$tables = $db.Tables + } + + # Filter the tables + if ($Table) { + [array]$tables = $tables | Where-Object { $_.Name -in $Table } + } + + # Loop through each of the tables + foreach ($tableObject in $tables) { + Write-Message -Message "Processing table [$($tableObject.Schema)].[$($tableObject.Name)]" -Level Verbose + + # Get the indexes + if ($Unique) { + [array]$indexes = $tableObject.Indexes | Where-Object { $_.IsUnique -eq $true } + } else { + [array]$indexes = $tableObject.Indexes + } + + # Get the indexed columns + [array]$indexedColumns = $indexes.IndexedColumns | Select-Object -ExpandProperty Name + + # Based on the indexed columns, get the actual column from the table object + [array]$columns = $tableObject.Columns | Where-Object { $_.Name -in $indexedColumns -and $_.Identity -eq $false } | Select-Object -Unique + + $columnStatements = @() + + foreach ($columnObject in $columns) { + # Check for user defined data types + if ($columnObject.DataType.SqlDataType -eq 'UserDefinedDataType') { + $uddt = $db.UserDefinedDataTypes[$columnObject.DataType.Name] + + [string]$dataType = $uddt.SystemType.ToLower().Trim() + [int]$length = $uddt.Length + } else { + [string]$dataType = $columnObject.DataType.SqlDataType.ToString().ToLower() + [int]$length = $columnObject.DataType.MaximumLength + } + + # Based on the data type create a different column statement + switch ($dataType) { + { $_ -in "bigint", "date", "datetime", "datetime2", "smallint", "time", "tinyint" } { + $columnStatements += "[$($columnObject.Name)] [$dataType]" + } + { $_ -like "*char*" } { + $columnStatements += "[$($columnObject.Name)] [$dataType]($length)" + } + default { + $columnStatements += "[$($columnObject.Name)] [$dataType]" + } + } + } + + # Add the id in there + $columnStatements += "[RowNr] [bigint]" + + # The query + if ($columns.Count -ge 1) { + [array]$columnNames = $columns.Name + $columnNames += "RowNr" + + $tableStatements += [PSCustomObject]@{ + Schema = "$($tableObject.Schema)" + Table = "$($tableObject.Name)" + Columns = $columnNames + TempTableName = "$($tableObject.Schema)_$($tableObject.Name)" + CreateStatement = "CREATE TABLE $($tableObject.Schema)_$($tableObject.Name)($($columnStatements -join ","));" + UniqueIndexName = "UIX_$($tableobject.Schema)_$($tableobject.Name)" + UniqueIndexStatement = "CREATE UNIQUE NONCLUSTERED INDEX [UIX_$($tableobject.Schema)_$($tableobject.Name)] ON $($tableObject.Schema)_$($tableObject.Name)([$($columnNames -join '],[')] ASC);" + } + } + } + } + + end { + if (Test-FunctionInterrupt) { return } + + $tableStatements + } +} + #.ExternalHelp dbatools-Help.xml function Convert-DbaLSN { @@ -76386,6 +76761,132 @@ function Convert-DbaLSN { } } +#.ExternalHelp dbatools-Help.xml +function Convert-DbaMaskingValue { + + + + param( + [Parameter(ValueFromPipeline)] + [AllowEmptyString()] + [object[]]$Value, + [string]$DataType, + [switch]$Nullable, + [switch]$EnableException + ) + + begin { + if (-not $Nullable -and -not $Value) { + Stop-Function -Message "Please enter a value" -Target $Value -Continue + } + + if (-not $Nullable -and -not $DataType) { + Stop-Function -Message "Please enter a data type" -Target $DataType -Continue + } + + if ($Value.Count -eq 0 -and $Nullable) { + [PSCustomObject]@{ + OriginalValue = '$null' + NewValue = 'NULL' + DataType = $DataType + ErrorMessage = $null + } + } + } + + process { + if (Test-FunctionInterrupt) { return } + + foreach ($item in $Value) { + + $originalValue = $item + + [string]$newValue = $null + [string]$errorMessage = $null + + if ($null -eq $item -or -not $item) { + $originalValue = '$null' + $newValue = "NULL" + } else { + switch ($DataType.ToLower()) { + { $_ -in 'bit', 'bool' } { + if ($item -match "([0-1])") { + + $newValue = "$item" + } else { + $errorMessage = "Value '$($item)' is not valid BIT or BOOL" + } + } + { $_ -like '*int*' -or $_ -in 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real' } { + if ($item -match "\b\d+([\.,]\d+)?") { + $newValue = "$item" + } else { + $errorMessage = "Value '$($item)' is not valid integer/decimal format" + } + } + { $_ -in 'uniqueidentifier' } { + $newValue = "'$item'" + } + { $_ -eq 'datetime' } { + if ($item -match "(\d{4})-(\d{2})-(\d{2})") { + $item = ([datetime]$item).Tostring("yyyy-MM-dd HH:mm:ss.fff") + $newValue = "'$item'" + } else { + $errorMessage = "Value '$($item)' is not valid DATE or DATETIME format (yyyy-MM-dd)" + } + } + { $_ -eq 'datetime2' } { + if ($item -match "(\d{4})-(\d{2})-(\d{2})") { + $item = ([datetime]$item).Tostring("yyyy-MM-dd HH:mm:ss.fffffff") + $newValue = "'$item'" + } else { + $errorMessage = "Value '$($item)' is not valid DATE or DATETIME format (yyyy-MM-dd)" + } + } + { $_ -eq 'date' } { + if ($item -match "(\d{4})-(\d{2})-(\d{2})") { + $item = ([datetime]$item).Tostring("yyyy-MM-dd") + $newValue = "'$item'" + } else { + $errorMessage = "Value '$($item)' is not valid DATE or DATETIME format (yyyy-MM-dd)" + } + } + { $_ -like 'smalldatetime' } { + if ($item -match "(\d{4})-(\d{2})-(\d{2})") { + $item = ([datetime]$item).Tostring("yyyy-MM-dd HH:mm:ss") + $newValue = "'$item'" + } else { + $errorMessage = "Value '$($item)' is not valid DATE or DATETIME format (yyyy-MM-dd)" + } + } + { $_ -eq 'time' } { + if ($item -match "(\d{2}):(\d{2}):(\d{2})") { + $item = ([datetime]$item).Tostring("HH:mm:ss.fffffff") + $newValue = "'$item'" + } else { + $errorMessage = "Value '$($item)' is not valid TIME format (HH:mm:ss)" + } + } + { $_ -eq 'xml' } { + # nothing, unsure how i'll handle this + } + default { + $item = ($item).Tostring().Replace("'", "''") + $newValue = "'$item'" + } + } + } + + [PSCustomObject]@{ + OriginalValue = $originalValue + NewValue = $newValue + DataType = $DataType + ErrorMessage = $errorMessage + } + } + } +} + #.ExternalHelp dbatools-Help.xml function Convert-DbaTimelineStatusColor { diff --git a/bin/dbatools-index.json b/bin/dbatools-index.json index 67881e6352..48cbf4e1b0 100644 --- a/bin/dbatools-index.json +++ b/bin/dbatools-index.json @@ -26526,7 +26526,7 @@ "", false, "false", - "10" + "0" ], [ "ExactLength", @@ -26550,7 +26550,7 @@ "", false, "false", - "300" + "0" ], [ "BatchSize", @@ -26558,7 +26558,15 @@ "", false, "false", - "1000" + "0" + ], + [ + "Retry", + "The amount of retries to generate a unique row for a table. Default is 1000.", + "", + false, + "false", + "0" ], [ "DictionaryFilePath", @@ -26601,7 +26609,7 @@ "" ] ], - "Syntax": "Invoke-DbaDbDataMasking [[-SqlInstance] \u003cDbaInstanceParameter[]\u003e] [[-SqlCredential] \u003cPSCredential\u003e] [[-Database] \u003cString[]\u003e] [-FilePath] \u003cObject\u003e [[-Locale] \u003cString\u003e] [[-CharacterString] \u003cString\u003e] [[-Table] \u003cString[]\u003e] [[-Column] \u003cString[]\u003e] [[-ExcludeTable] \u003cString[]\u003e] [[-ExcludeColumn] \u003cString[]\u003e] [[-MaxValue] \u003cInt32\u003e] [[-ModulusFactor] \u003cInt32\u003e] [-ExactLength] [[-ConnectionTimeout] \u003cInt32\u003e] [[-CommandTimeout] \u003cInt32\u003e] [[-BatchSize] \u003cInt32\u003e] [[-DictionaryFilePath] \u003cString[]\u003e] [[-DictionaryExportPath] \u003cString\u003e] [-EnableException] [-WhatIf] [-Confirm] [\u003cCommonParameters\u003e]" + "Syntax": "Invoke-DbaDbDataMasking [[-SqlInstance] \u003cDbaInstanceParameter[]\u003e] [[-SqlCredential] \u003cPSCredential\u003e] [[-Database] \u003cString[]\u003e] [-FilePath] \u003cObject\u003e [[-Locale] \u003cString\u003e] [[-CharacterString] \u003cString\u003e] [[-Table] \u003cString[]\u003e] [[-Column] \u003cString[]\u003e] [[-ExcludeTable] \u003cString[]\u003e] [[-ExcludeColumn] \u003cString[]\u003e] [[-MaxValue] \u003cInt32\u003e] [[-ModulusFactor] \u003cInt32\u003e] [-ExactLength] [[-ConnectionTimeout] \u003cInt32\u003e] [[-CommandTimeout] \u003cInt32\u003e] [[-BatchSize] \u003cInt32\u003e] [[-Retry] \u003cInt32\u003e] [[-DictionaryFilePath] \u003cString[]\u003e] [[-DictionaryExportPath] \u003cString\u003e] [-EnableException] [-WhatIf] [-Confirm] [\u003cCommonParameters\u003e]" }, { "Name": "Invoke-DbaDbDbccCheckConstraint", @@ -48759,15 +48767,15 @@ }, { "Name": "Test-DbaDiskSpeed", - "Description": "Tests how disks are performing.\n\nThis command uses a query from Rich Benner\nhttps://github.com/RichBenner/PersonalCode/blob/master/Disk_Speed_Check.sql", + "Description": "Obtains I/O statistics based on the DMV sys.dm_io_virtual_file_stats:\n\nhttps://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-io-virtual-file-stats-transact-sql\n\nThis command uses a query from Rich Benner\nhttps://github.com/RichBenner/PersonalCode/blob/master/Disk_Speed_Check.sql\n\n...and also based on further adaptations referenced at https://github.com/sqlcollaborative/dbatools/issues/6551#issue-623216718", "Tags": "Performance", - "Synopsis": "Tests how disks are performing.", + "Synopsis": "Obtains I/O statistics based on the DMV sys.dm_io_virtual_file_stats:\n\nhttps://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-io-virtual-file-stats-transact-sql", "Alias": "", "Author": "Chrissy LeMaire (@cl), netnerds.net", "CommandName": "Test-DbaDiskSpeed", "Availability": "Windows, Linux, macOS", "Links": "https://dbatools.io/Test-DbaDiskSpeed", - "Examples": "-------------------------- EXAMPLE 1 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008, sqlserver2012\nTests how disks are performing on sql2008 and sqlserver2012.\n-------------------------- EXAMPLE 2 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008 -Database tempdb\nTests how disks storing tempdb files on sql2008 are performing.", + "Examples": "-------------------------- EXAMPLE 1 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008, sqlserver2012\nTests how disks are performing on sql2008 and sqlserver2012.\n-------------------------- EXAMPLE 2 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008 -Database tempdb\nTests how disks storing tempdb files on sql2008 are performing.\n-------------------------- EXAMPLE 3 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008 -AggregateBy \"File\" -Database tempdb\nReturns the statistics aggregated to the file level. This is the default aggregation level if the -AggregateBy param is omitted. The -Database or -ExcludeDatabase params can be used to filter for \r\nspecific databases.\n-------------------------- EXAMPLE 4 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008 -AggregateBy \"Database\"\nReturns the statistics aggregated to the database/disk level. The -Database or -ExcludeDatabase params can be used to filter for specific databases.\n-------------------------- EXAMPLE 5 --------------------------\nPS C:\\\u003eTest-DbaDiskSpeed -SqlInstance sql2008 -AggregateBy \"Disk\"\nReturns the statistics aggregated to the disk level. The -Database or -ExcludeDatabase params can be used to filter for specific databases.\n-------------------------- EXAMPLE 6 --------------------------\nPS C:\\\u003e$results = @(instance1, instance2) | Test-DbaDiskSpeed\nReturns the statistics for instance1 and instance2 as part of a pipeline command\n-------------------------- EXAMPLE 7 --------------------------\nPS C:\\\u003e$databases = @(\u0027master\u0027, \u0027model\u0027)\n$results = Test-DbaDiskSpeed -SqlInstance sql2019 -Database $databases\nReturns the statistics for more than one database specified.\n-------------------------- EXAMPLE 8 --------------------------\nPS C:\\\u003e$excludedDatabases = @(\u0027master\u0027, \u0027model\u0027)\n$results = Test-DbaDiskSpeed -SqlInstance sql2019 -ExcludeDatabase $excludedDatabases\nReturns the statistics for databases other than the exclusions specified.", "Params": [ [ "SqlInstance", @@ -48801,6 +48809,14 @@ "false", "" ], + [ + "AggregateBy", + "Specify the level of aggregation for the statistics. The available options are \u0027File\u0027 (the default), \u0027Database\u0027, and \u0027Disk\u0027.", + "", + false, + "false", + "File" + ], [ "EnableException", "By default, when something goes wrong we try to catch it, interpret it and give you a friendly warning message.\r\nThis avoids overwhelming you with \"sea of red\" exceptions, but is inconvenient because it basically disables advanced scripting.\r\nUsing this switch turns this \"nice by default\" feature off and enables you to catch exceptions with your own try/catch.", @@ -48810,7 +48826,7 @@ "False" ] ], - "Syntax": "Test-DbaDiskSpeed [-SqlInstance] \u003cDbaInstanceParameter[]\u003e [[-SqlCredential] \u003cPSCredential\u003e] [[-Database] \u003cObject[]\u003e] [[-ExcludeDatabase] \u003cObject[]\u003e] [-EnableException] [\u003cCommonParameters\u003e]" + "Syntax": "Test-DbaDiskSpeed [-SqlInstance] \u003cDbaInstanceParameter[]\u003e [[-SqlCredential] \u003cPSCredential\u003e] [[-Database] \u003cObject[]\u003e] [[-ExcludeDatabase] \u003cObject[]\u003e] [[-AggregateBy] \u003cString\u003e] [-EnableException] [\u003cCommonParameters\u003e]" }, { "Name": "Test-DbaEndpoint", diff --git a/dbatools.psd1 b/dbatools.psd1 index 3d5b65765b..ba029b612d 100644 --- a/dbatools.psd1 +++ b/dbatools.psd1 @@ -11,7 +11,7 @@ RootModule = 'dbatools.psm1' # Version number of this module. - ModuleVersion = '1.0.123' + ModuleVersion = '1.0.124' # ID used to uniquely identify this module GUID = '9d139310-ce45-41ce-8e8b-d76335aa1789' diff --git a/en-us/dbatools-help.xml b/en-us/dbatools-help.xml index f592554678..4a181fdb46 100644 --- a/en-us/dbatools-help.xml +++ b/en-us/dbatools-help.xml @@ -7933,7 +7933,7 @@ Windows Authentication, SQL Server Authentication, Active Directory - Password, For MFA support, please use Connect-DbaInstance.PscredentialPscredentialDatabaseDatabases to process throughSystem.String[]System.String[]FilePathConfiguration file that contains the which tables and columns need to be maskedSystem.ObjectSystem.ObjectLocaleSet the local to enable certain settings in the maskingStringStringCharacterStringThe characters to use in string data. 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' by defaultStringStringTableTables to process. By default all the tables will be processedSystem.String[]System.String[]ColumnColumns to process. By default all the columns will be processedSystem.String[]System.String[]ExcludeTableExclude specific tables even if it's listed in the config file.System.String[]System.String[]ExcludeColumnExclude specific columns even if it's listed in the config file.System.String[]System.String[]MaxValueForce a max length of strings instead of relying on datatype maxes. Note if a string datatype has a lower MaxValue, that will be used instead. -Useful for adhoc updates and testing, otherwise, the config file should be used.IntIntModulusFactorCalculating the next nullable by using the remainder from the modulus. Default is every 10.IntIntExactLengthMask string values to the same length. So 'Tate' will be replaced with 4 random characters.SwitchSwitchConnectionTimeoutTimeout for the database connection in seconds. Default is 0IntIntCommandTimeoutTimeout for the database connection in seconds. Default is 300.IntIntBatchSizeSize of the batch to use to write the masked data back to the databaseIntIntDictionaryFilePathImport the dictionary to be used in in the database maskingSystem.String[]System.String[]DictionaryExportPathExport the dictionary to the given path. Naming convention will be [computername]_[instancename]_[database]_Dictionary.csv +Useful for adhoc updates and testing, otherwise, the config file should be used.IntIntModulusFactorCalculating the next nullable by using the remainder from the modulus. Default is every 10.IntIntExactLengthMask string values to the same length. So 'Tate' will be replaced with 4 random characters.SwitchSwitchConnectionTimeoutTimeout for the database connection in seconds. Default is 0IntIntCommandTimeoutTimeout for the database connection in seconds. Default is 300.IntIntBatchSizeSize of the batch to use to write the masked data back to the databaseIntIntRetryThe amount of retries to generate a unique row for a table. Default is 1000.IntIntDictionaryFilePathImport the dictionary to be used in in the database maskingSystem.String[]System.String[]DictionaryExportPathExport the dictionary to the given path. Naming convention will be [computername]_[instancename]_[database]_Dictionary.csv Be careful with this feature, this export is the key to get the original values which is a security risk!StringStringEnableExceptionBy default, when something goes wrong we try to catch it, interpret it and give you a friendly warning message. This avoids overwhelming you with "sea of red" exceptions, but is inconvenient because it basically disables advanced scripting. @@ -7943,7 +7943,7 @@ Be careful with this feature, this export is the key to get the original values This avoids overwhelming you with "sea of red" exceptions, but is inconvenient because it basically disables advanced scripting. Using this switch turns this "nice by default" feature off and enables you to catch exceptions with your own try/catch.SwitchSwitchExactLengthMask string values to the same length. So 'Tate' will be replaced with 4 random characters.SwitchSwitchExcludeColumnExclude specific columns even if it's listed in the config file.System.String[]System.String[]ExcludeTableExclude specific tables even if it's listed in the config file.System.String[]System.String[]FilePathConfiguration file that contains the which tables and columns need to be maskedSystem.ObjectSystem.ObjectLocaleSet the local to enable certain settings in the maskingStringStringMaxValueForce a max length of strings instead of relying on datatype maxes. Note if a string datatype has a lower MaxValue, that will be used instead. -Useful for adhoc updates and testing, otherwise, the config file should be used.IntIntModulusFactorCalculating the next nullable by using the remainder from the modulus. Default is every 10.IntIntSqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). +Useful for adhoc updates and testing, otherwise, the config file should be used.IntIntModulusFactorCalculating the next nullable by using the remainder from the modulus. Default is every 10.IntIntRetryThe amount of retries to generate a unique row for a table. Default is 1000.IntIntSqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). Windows Authentication, SQL Server Authentication, Active Directory - Password, and Active Directory - Integrated are all supported. @@ -13740,13 +13740,15 @@ Author: Chrissy LeMaire (@cl), netnerds.net Website: https://dbatools.io Copyright: (c) 2018 by dbatools, licensed under MIT -License: MIT https://opensource.org/licenses/MIT -------------------------- EXAMPLE 1 -------------------------- PS C:\> Test-DbaDiskAllocation -ComputerName sqlserver2014a Scans all disks on server sqlserver2014a for best practice allocation unit size. -------------------------- EXAMPLE 2 -------------------------- PS C:\> Test-DbaDiskAllocation -ComputerName sqlserver2014 | Select-Output * Scans all disks on server sqlserver2014a for allocation unit size and returns detailed results for each. -------------------------- EXAMPLE 3 -------------------------- PS C:\> Test-DbaDiskAllocation -ComputerName sqlserver2014a -NoSqlCheck Scans all disks not hosting SQL Server data or log files on server sqlserver2014a for best practice allocation unit size. https://dbatools.io/Test-DbaDiskAllocationTest-DbaDiskSpeedDbaDiskSpeedTestTests how disks are performing.Tests how disks are performing.This command uses a query from Rich Bennerhttps://github.com/RichBenner/PersonalCode/blob/master/Disk_Speed_Check.sqlTest-DbaDiskSpeedSqlInstanceThe target SQL Server instance or instances.Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]SqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). +License: MIT https://opensource.org/licenses/MIT -------------------------- EXAMPLE 1 -------------------------- PS C:\> Test-DbaDiskAllocation -ComputerName sqlserver2014a Scans all disks on server sqlserver2014a for best practice allocation unit size. -------------------------- EXAMPLE 2 -------------------------- PS C:\> Test-DbaDiskAllocation -ComputerName sqlserver2014 | Select-Output * Scans all disks on server sqlserver2014a for allocation unit size and returns detailed results for each. -------------------------- EXAMPLE 3 -------------------------- PS C:\> Test-DbaDiskAllocation -ComputerName sqlserver2014a -NoSqlCheck Scans all disks not hosting SQL Server data or log files on server sqlserver2014a for best practice allocation unit size. https://dbatools.io/Test-DbaDiskAllocationTest-DbaDiskSpeedDbaDiskSpeedTestObtains I/O statistics based on the DMV sys.dm_io_virtual_file_stats: + +https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-io-virtual-file-stats-transact-sqlObtains I/O statistics based on the DMV sys.dm_io_virtual_file_stats:https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-io-virtual-file-stats-transact-sqlThis command uses a query from Rich Bennerhttps://github.com/RichBenner/PersonalCode/blob/master/Disk_Speed_Check.sql...and also based on further adaptations referenced at https://github.com/sqlcollaborative/dbatools/issues/6551#issue-623216718Test-DbaDiskSpeedSqlInstanceThe target SQL Server instance or instances.Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]SqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). Windows Authentication, SQL Server Authentication, Active Directory - Password, and Active Directory - Integrated are all supported. -For MFA support, please use Connect-DbaInstance.PscredentialPscredentialDatabaseThe database(s) to process - this list is auto-populated from the server. If unspecified, all databases will be processed.System.Object[]System.Object[]ExcludeDatabaseThe database(s) to exclude - this list is auto-populated from the serverSystem.Object[]System.Object[]EnableExceptionBy default, when something goes wrong we try to catch it, interpret it and give you a friendly warning message. +For MFA support, please use Connect-DbaInstance.PscredentialPscredentialDatabaseThe database(s) to process - this list is auto-populated from the server. If unspecified, all databases will be processed.System.Object[]System.Object[]ExcludeDatabaseThe database(s) to exclude - this list is auto-populated from the serverSystem.Object[]System.Object[]AggregateBySpecify the level of aggregation for the statistics. The available options are 'File' (the default), 'Database', and 'Disk'.StringStringEnableExceptionBy default, when something goes wrong we try to catch it, interpret it and give you a friendly warning message. This avoids overwhelming you with "sea of red" exceptions, but is inconvenient because it basically disables advanced scripting. -Using this switch turns this "nice by default" feature off and enables you to catch exceptions with your own try/catch.SwitchSwitchDatabaseThe database(s) to process - this list is auto-populated from the server. If unspecified, all databases will be processed.System.Object[]System.Object[]EnableExceptionBy default, when something goes wrong we try to catch it, interpret it and give you a friendly warning message. +Using this switch turns this "nice by default" feature off and enables you to catch exceptions with your own try/catch.SwitchSwitchAggregateBySpecify the level of aggregation for the statistics. The available options are 'File' (the default), 'Database', and 'Disk'.StringStringDatabaseThe database(s) to process - this list is auto-populated from the server. If unspecified, all databases will be processed.System.Object[]System.Object[]EnableExceptionBy default, when something goes wrong we try to catch it, interpret it and give you a friendly warning message. This avoids overwhelming you with "sea of red" exceptions, but is inconvenient because it basically disables advanced scripting. Using this switch turns this "nice by default" feature off and enables you to catch exceptions with your own try/catch.SwitchSwitchExcludeDatabaseThe database(s) to exclude - this list is auto-populated from the serverSystem.Object[]System.Object[]SqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). @@ -13757,7 +13759,11 @@ Author: Chrissy LeMaire (@cl), netnerds.net Website: https://dbatools.io Copyright: (c) 2018 by dbatools, licensed under MIT -License: MIT https://opensource.org/licenses/MIT -------------------------- EXAMPLE 1 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008, sqlserver2012 Tests how disks are performing on sql2008 and sqlserver2012. -------------------------- EXAMPLE 2 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008 -Database tempdb Tests how disks storing tempdb files on sql2008 are performing. https://dbatools.io/Test-DbaDiskSpeedTest-DbaEndpointDbaEndpointTestPerforms a simple connectivity test for TCP and SSL enabled endpoints.Performs a simple connectivity test for TCP and SSL enabled endpoints. Tests if port is accessible, not if endpoint is working.Note that if an endpoint does not have a tcp listener port, it will be skipped.Test-DbaEndpointSqlInstanceThe target SQL Server instance or instances.Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]SqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). +License: MIT https://opensource.org/licenses/MIT -------------------------- EXAMPLE 1 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008, sqlserver2012 Tests how disks are performing on sql2008 and sqlserver2012. -------------------------- EXAMPLE 2 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008 -Database tempdb Tests how disks storing tempdb files on sql2008 are performing. -------------------------- EXAMPLE 3 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008 -AggregateBy "File" -Database tempdb Returns the statistics aggregated to the file level. This is the default aggregation level if the -AggregateBy param is omitted. The -Database or -ExcludeDatabase params can be used to filter for specific databases. -------------------------- EXAMPLE 4 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008 -AggregateBy "Database" Returns the statistics aggregated to the database/disk level. The -Database or -ExcludeDatabase params can be used to filter for specific databases. -------------------------- EXAMPLE 5 -------------------------- PS C:\> Test-DbaDiskSpeed -SqlInstance sql2008 -AggregateBy "Disk" Returns the statistics aggregated to the disk level. The -Database or -ExcludeDatabase params can be used to filter for specific databases. -------------------------- EXAMPLE 6 -------------------------- PS C:\> $results = @(instance1, instance2) | Test-DbaDiskSpeed Returns the statistics for instance1 and instance2 as part of a pipeline command -------------------------- EXAMPLE 7 -------------------------- PS C:\> $databases = @('master', 'model') $results = Test-DbaDiskSpeed -SqlInstance sql2019 -Database $databases + +Returns the statistics for more than one database specified. -------------------------- EXAMPLE 8 -------------------------- PS C:\> $excludedDatabases = @('master', 'model') $results = Test-DbaDiskSpeed -SqlInstance sql2019 -ExcludeDatabase $excludedDatabases + +Returns the statistics for databases other than the exclusions specified. https://dbatools.io/Test-DbaDiskSpeedTest-DbaEndpointDbaEndpointTestPerforms a simple connectivity test for TCP and SSL enabled endpoints.Performs a simple connectivity test for TCP and SSL enabled endpoints. Tests if port is accessible, not if endpoint is working.Note that if an endpoint does not have a tcp listener port, it will be skipped.Test-DbaEndpointSqlInstanceThe target SQL Server instance or instances.Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]Sqlcollaborative.Dbatools.Parameter.DbaInstanceParameter[]SqlCredentialLogin to the target instance using alternative credentials. Accepts PowerShell credentials (Get-Credential). Windows Authentication, SQL Server Authentication, Active Directory - Password, and Active Directory - Integrated are all supported.