Skip to content

Commit

Permalink
find-dbaorphaned, new recurse switch (#6893)
Browse files Browse the repository at this point in the history
  • Loading branch information
@niphlod
niphlod authored Oct 1, 2020
1 parent fa90656 commit 018d02e
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 69 deletions.
150 changes: 96 additions & 54 deletions functions/Find-DbaOrphanedFile.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ function Find-DbaOrphanedFile {
This avoids overwhelming you with "sea of red" exceptions, but is inconvenient because it basically disables advanced scripting.
Using this switch turns this "nice by default" feature off and enables you to catch exceptions with your own try/catch.
.PARAMETER Recurse
If this switch is enabled, the command will search subdirectories of the Path parameter.
.NOTES
Tags: Orphan, Database, DatabaseFile
Author: Sander Stad (@sqlstad), sqlstad.nl
Expand Down Expand Up @@ -67,6 +70,11 @@ function Find-DbaOrphanedFile {
Finds the orphaned files in "E:\Dir1" and "E:Dir2" in addition to the default directories.
.EXAMPLE
PS C:\> Find-DbaOrphanedFile -SqlInstance sql2014 -Path 'E:\Dir1' -Recurse
Finds the orphaned files in "E:\Dir1" and any of its subdirectories in addition to the default directories.
.EXAMPLE
PS C:\> Find-DbaOrphanedFile -SqlInstance sql2014 -LocalOnly
Expand All @@ -81,55 +89,85 @@ function Find-DbaOrphanedFile {
PS C:\> Find-DbaOrphanedFile -SqlInstance sql2014, sql2016 -FileType fsf, mld
Finds the orphaned ending with ".fsf" and ".mld" in addition to the default filetypes ".mdf", ".ldf", ".ndf" for both the servers sql2014 and sql2016.
#>
[CmdletBinding()]

[CmdletBinding(DefaultParameterSetName = 'LocalOnly')]

param (
[parameter(Mandatory, ValueFromPipeline)]
[Parameter(Mandatory, ValueFromPipeline)]
[DbaInstanceParameter[]]$SqlInstance,
[pscredential]$SqlCredential,
[string[]]$Path,
[string[]]$FileType,
[switch]$LocalOnly,
[switch]$RemoteOnly,
[switch]$EnableException
[Parameter(ParameterSetName = 'LocalOnly')][switch]$LocalOnly,
[Parameter(ParameterSetName = 'RemoteOnly')][switch]$RemoteOnly,
[switch]$EnableException,
[switch]$Recurse
)

begin {
function Get-SQLDirTreeQuery {
param($PathList)
# use sysaltfiles in lower versions

$q1 = "CREATE TABLE #enum ( id int IDENTITY, fs_filename nvarchar(512), depth int, is_file int, parent nvarchar(512) ); DECLARE @dir nvarchar(512);"
$q2 = "SET @dir = 'dirname';
param([object[]]$SqlPathList, [object[]]$UserPathList, $FileTypes, $SystemFiles, [Switch]$Recurse)

$q1 = "
CREATE TABLE #enum (
id int IDENTITY
, fs_filename nvarchar(512)
, depth int
, is_file int
, parent nvarchar(512)
, parent_id int
);
DECLARE @dir nvarchar(512);
"

$q2 = "
SET @dir = 'dirname';
INSERT INTO #enum( fs_filename, depth, is_file )
EXEC xp_dirtree @dir, 1, 1;
EXEC xp_dirtree @dir, recurse, 1;
UPDATE #enum
SET parent = @dir,
fs_filename = ltrim(rtrim(fs_filename))
WHERE parent IS NULL;"

$query_files_sql = "SELECT e.fs_filename AS filename, e.parent
FROM #enum AS e
WHERE e.fs_filename NOT IN( 'xtp', '5', '`$FSLOG', '`$HKv2', 'filestream.hdr' )
AND is_file = 1;"
fs_filename = ltrim(rtrim(e.fs_filename)),
parent_id = (SELECT MAX(i.id) FROM #enum i WHERE i.id < e.id AND i.depth = e.depth-1 AND i.is_file = 0)
FROM #enum e
WHERE e.parent IS NULL;
"

$query_files_sql = "
SELECT e.fs_filename AS filename, e.parent
FROM #enum AS e
WHERE e.fs_filename NOT IN( 'xtp', '5', '`$FSLOG', '`$HKv2', 'filestream.hdr', '" + $($SystemFiles -join "','") + "' )
AND CASE
WHEN e.fs_filename LIKE '%.%'
THEN REVERSE(LEFT(REVERSE(e.fs_filename), CHARINDEX('.', REVERSE(e.fs_filename)) - 1))
ELSE ''
END IN('" + $($FileTypes -join "','") + "')
AND e.is_file = 1;
"

# build the query string based on how many directories they want to enumerate
$sql = $q1
$sql += $($PathList | Where-Object { $_ -ne '' } | ForEach-Object { "$([System.Environment]::Newline)$($q2 -Replace 'dirname', $_)" })
$sql += $($SqlPathList | Where-Object { $_ -ne '' } | ForEach-Object { "$([System.Environment]::Newline)$($q2.Replace('dirname',$_).Replace('recurse','1'))" } )
If ($UserPathList) {
$recurseVal = If ($Recurse) { '0' } Else { '1' }
$sql += $($UserPathList | Where-Object { $_ -ne '' } | ForEach-Object { "$([System.Environment]::Newline)$($q2.Replace('dirname',$_).Replace('recurse',$recurseVal))" } )
}
$sql += $query_files_sql
Write-Message -Level Debug -Message $sql
return $sql
}

function Get-SqlFileStructure {
param
(
[Parameter(Mandatory, Position = 1)]
[Microsoft.SqlServer.Management.Smo.SqlSmoObject]$smoserver
)
if ($smoserver.versionMajor -eq 8) {

# use sysaltfiles in lower versions
if ($smoserver.VersionMajor -eq 8) {
$sql = "select filename from sysaltfiles"
} else {
$sql = "select physical_name as filename from sys.master_files"
Expand All @@ -141,20 +179,17 @@ function Find-DbaOrphanedFile {

# Add support for Full Text Catalogs in Sql Server 2005 and below
if ($server.VersionMajor -lt 10) {
$databaselist = $smoserver.Databases | Select-Object -property Name, IsFullTextEnabled
foreach ($db in $databaselist) {
if ($db.IsFullTextEnabled -eq $false) {
continue
}
$database = $db.name
$databaselist = $smoserver.Databases | Select-Object -Property Name, IsFullTextEnabled
foreach ($db in $databaselist | Where-Object IsFullTextEnabled) {
$database = $db.Name
$fttable = $null = $smoserver.Databases[$database].ExecuteWithResults('sp_help_fulltext_catalogs')
foreach ($ftc in $fttable.Tables[0].rows) {
$null = $ftfiletable.Rows.add($ftc.Path)
foreach ($ftc in $fttable.Tables[0].Rows) {
$null = $ftfiletable.Rows.Add($ftc.Path)
}
}
}

$null = $dbfiletable.Tables.Add($ftfiletable)

return $dbfiletable.Tables.Filename
}

Expand All @@ -167,51 +202,59 @@ function Find-DbaOrphanedFile {
return $path
}

$FileType += "mdf", "ldf", "ndf"
$systemfiles = "distmdl.ldf", "distmdl.mdf", "mssqlsystemresource.ldf", "mssqlsystemresource.mdf", "model_msdbdata.mdf", "model_msdblog.ldf", "model_replicatedmaster.mdf", "model_replicatedmaster.ldf"

$FileTypeComparison = $FileType | ForEach-Object { $_.ToLowerInvariant() } | Where-Object { $_ } | Sort-Object | Get-Unique
$FileType += "mdf", "ldf", "ndf"
$fileTypeComparison = $FileType | ForEach-Object { $_.ToLowerInvariant() } | Where-Object { $_ } | Sort-Object -Unique
}

process {
foreach ($instance in $SqlInstance) {

# Connect to the instance
try {
$server = Connect-SqlInstance -SqlInstance $instance -SqlCredential $sqlcredential
} catch {
Stop-Function -Message "Error occurred while establishing connection to $instance" -Category ConnectionError -ErrorRecord $_ -Target $instance -Continue
}

# Reset all the arrays
$dirtreefiles = $valid = $paths = $matching = @()
$sqlpaths = $userpaths = $matching = $valid = @()
$dirtreefiles = @{ }

$filestructure = Get-SqlFileStructure $server
# Gather a list of files known to SQL Server
$sqlfiles = Get-SqlFileStructure $server

# Get any paths associated with current data and log files
foreach ($file in $filestructure) {
$paths += Split-Path -Path $file -Parent
# Get the parent directories of those files
$sqlfiles | ForEach-Object {
$sqlpaths += Split-Path -Path $_ -Parent
}

# Get the default data and log directories from the instance
# Include the default data and log directories from the instance
Write-Message -Level Debug -Message "Adding paths"
$paths += $server.RootDirectory + "\DATA"
$paths += Get-SqlDefaultPaths $server data
$paths += Get-SqlDefaultPaths $server log
$paths += $server.MasterDBPath
$paths += $server.MasterDBLogPath
$paths += $Path
$paths = $paths | ForEach-Object { "$_".TrimEnd("\") } | Sort-Object | Get-Unique
$sql = Get-SQLDirTreeQuery $paths
$datatable = $server.Databases['master'].ExecuteWithResults($sql).Tables[0]

foreach ($row in $datatable) {
$fullpath = [IO.Path]::combine($row.parent, $row.filename)
$dirtreefiles += [pscustomobject]@{
$sqlpaths += "$($server.RootDirectory)\DATA"
$sqlpaths += Get-SqlDefaultPaths $server data
$sqlpaths += Get-SqlDefaultPaths $server log
$sqlpaths += $server.MasterDBPath
$sqlpaths += $server.MasterDBLogPath

# Gather a list of files from the filesystem
$sqlpaths = $sqlpaths | ForEach-Object { $_.TrimEnd("\") } | Sort-Object -Unique
if ($Path) {
$userpaths = $Path | ForEach-Object { $_.TrimEnd("\") } | Sort-Object -Unique
}
$sql = Get-SQLDirTreeQuery -SqlPathList $sqlpaths -UserPathList $userpaths -FileTypes $fileTypeComparison -SystemFiles $systemfiles -Recurse:$Recurse
$dirtreefiles = $server.Databases['master'].ExecuteWithResults($sql).Tables[0] | ForEach-Object {
$fullpath = [IO.Path]::combine($_.parent, $_.filename)
[PSCustomObject]@{
FullPath = $fullpath
Comparison = [IO.Path]::GetFullPath($(Format-Path $fullpath))
}
}
# Output files in the dirtree not known to SQL Server
$dirtreefiles = $dirtreefiles | Where-Object { $_ } | Sort-Object Comparison -Unique

foreach ($file in $filestructure) {
foreach ($file in $sqlfiles) {
$valid += [IO.Path]::GetFullPath($(Format-Path $file))
}

Expand All @@ -228,9 +271,8 @@ function Find-DbaOrphanedFile {

$dirtreematcher = @{ }
foreach ($el in $dirtreefiles) {
$dirtreematcher[$el.Comparison] = $el.Fullpath
$dirtreematcher[$el.Comparison] = $el.FullPath
}

foreach ($file in $matching) {
if ($file -notin $valid) {
$fullpath = $dirtreematcher[$file]
Expand Down
46 changes: 31 additions & 15 deletions tests/Find-DbaOrphanedFile.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ Write-Host -Object "Running $PSCommandPath" -ForegroundColor Cyan
Describe "$CommandName Unit Tests" -Tag 'UnitTests' {
Context "Validate parameters" {
[object[]]$params = (Get-Command $CommandName).Parameters.Keys | Where-Object {$_ -notin ('whatif', 'confirm')}
[object[]]$knownParameters = 'SqlInstance', 'SqlCredential', 'Path', 'FileType', 'LocalOnly', 'RemoteOnly', 'EnableException'
[object[]]$knownParameters = 'SqlInstance', 'SqlCredential', 'Path', 'FileType', 'LocalOnly', 'RemoteOnly', 'Recurse', 'EnableException'
$knownParameters += [System.Management.Automation.PSCmdlet]::CommonParameters
It "Should only contain our specific parameters" {
(@(Compare-Object -ReferenceObject ($knownParameters | Where-Object {$_}) -DifferenceObject $params).Count ) | Should Be 0
(@(Compare-Object -ReferenceObject ($knownParameters | Where-Object {$_}) -DifferenceObject $params).Count ) | Should -Be 0
}
}
}
Expand All @@ -19,6 +19,12 @@ Describe "$CommandName Integration Tests" -Tags "IntegrationTests" {
$dbname = "dbatoolsci_orphanedfile"
$server = Connect-DbaInstance -SqlInstance $script:instance2
$null = $server.Query("CREATE DATABASE $dbname")
$tmpdir = "c:\temp\orphan_$(Get-Random)"
if (-not(Test-Path $tmpdir)) {
$null = New-Item -Path $tmpdir -type Container
}
$tmpdirInner = Join-Path $tmpdir "inner"
$null = New-Item -Path $tmpdirInner -type Container
$result = Get-DbaDatabase -SqlInstance $script:instance2 -Database $dbname
if ($result.count -eq 0) {
it "has failed setup" {
Expand All @@ -29,28 +35,38 @@ Describe "$CommandName Integration Tests" -Tags "IntegrationTests" {
}
AfterAll {
Get-DbaDatabase -SqlInstance $script:instance2 -Database $dbname | Remove-DbaDatabase -Confirm:$false
}
$null = Detach-DbaDatabase -SqlInstance $script:instance2 -Database $dbname -Force
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2

It "Has the correct default properties" {
$ExpectedStdProps = 'ComputerName,InstanceName,SqlInstance,Filename,RemoteFilename'.Split(',')
($results[0].PSStandardMembers.DefaultDisplayPropertySet.ReferencedPropertyNames | Sort-Object) | Should Be ($ExpectedStdProps | Sort-Object)
Remove-Item $tmpdir -Recurse -Force -ErrorAction SilentlyContinue
}
It "Has the correct properties" {
$null = Detach-DbaDatabase -SqlInstance $script:instance2 -Database $dbname -Force
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2
$ExpectedStdProps = 'ComputerName,InstanceName,SqlInstance,Filename,RemoteFilename'.Split(',')
($results[0].PSStandardMembers.DefaultDisplayPropertySet.ReferencedPropertyNames | Sort-Object) | Should -Be ($ExpectedStdProps | Sort-Object)
$ExpectedProps = 'ComputerName,InstanceName,SqlInstance,Filename,RemoteFilename,Server'.Split(',')
($results[0].PsObject.Properties.Name | Sort-Object) | Should Be ($ExpectedProps | Sort-Object)
($results[0].PsObject.Properties.Name | Sort-Object) | Should -Be ($ExpectedProps | Sort-Object)
}


It "Finds two files" {
$results.Count | Should Be 2
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2
$results.Filename.Count | Should -Be 2
}

$results.FileName | Remove-Item

$results = Find-DbaOrphanedFile -SqlInstance $script:instance2
It "Finds zero files after cleaning up" {
$results.Count | Should Be 0
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2
$results.FileName | Remove-Item
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2
$results.Filename.Count | Should -Be 0
}
It "works with -Recurse" {
"a" | out-file (Join-Path $tmpdir "out.mdf")
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2 -Path $tmpdir
$results.Filename.Count | Should -Be 1
move-item "$tmpdir\out.mdf" -destination $tmpdirInner
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2 -Path $tmpdir
$results.Filename.Count | Should -Be 0
$results = Find-DbaOrphanedFile -SqlInstance $script:instance2 -Path $tmpdir -Recurse
$results.Filename.Count | Should -Be 1
}
}
}

0 comments on commit 018d02e

Please sign in to comment.