From e2384e560b3e43198b6ee6c19e13797c1c7ec997 Mon Sep 17 00:00:00 2001 From: Raimund Andree Date: Sat, 10 Oct 2026 06:55:41 +0000 Subject: [PATCH] test(lab): count and repair the profiles and log-group entries of the probe, and read the whole series Test-MatrixCleanup.ps1 now also counts and removes the profiles and the profile folders C:\Users\NtfsProbe* (retried, because a profile that the last task used stays loaded for a few seconds) and the entries of the account probe in Performance Log Users. net.exe lists a local user by its bare name and a deleted domain account by its SID or its cached name, so the check matches NtfsProbe anywhere in the line or a SID. Export-CellTimeline.ps1 takes the account of a cell that stopped before its tests from the snapshot of its fixture, so that the series can include such a cell, and reports SameNameAsPreviousCell and SameAccountAsPreviousCell instead of one column that compared names only. Test-StaleAuthzModel.ps1 computes its grid of lifetimes by index (the accumulated step lost the grid point 10.25), prints a range as segments, and prints the range of one lifetime for both tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: AI Assistant --- Tests/Lab/Acceptance/Export-CellTimeline.ps1 | 43 ++++++++---- Tests/Lab/Acceptance/Test-MatrixCleanup.ps1 | 65 +++++++++++++------ Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1 | 49 +++++++++++++- 3 files changed, 123 insertions(+), 34 deletions(-) diff --git a/Tests/Lab/Acceptance/Export-CellTimeline.ps1 b/Tests/Lab/Acceptance/Export-CellTimeline.ps1 index 2456ec4..844fe78 100644 --- a/Tests/Lab/Acceptance/Export-CellTimeline.ps1 +++ b/Tests/Lab/Acceptance/Export-CellTimeline.ps1 @@ -5,13 +5,14 @@ param ( [Parameter(Mandatory)] [string] $OutputPath ) -# The timeline of the cells of the operating-system matrix (Decision 24): for each cell and edition, in the order in which the cells ran, the module under -# test, the account of case 3 (its name and its relative ID, which tells two accounts of one name apart within a domain), when the previous fixture was -# removed, when the accounts were created, when the Admin role started, the minutes between -# them, and the three effective-access tests of case 3 in the Admin role (result, milliseconds, and the rights that a failing test received). A failing -# effective-access test of the Admin role is easy to blame on the module or on the environment; this table puts it beside the module, the position of -# the cell in the sequence, and the age of the accounts. The times come from the logs of Run-MatrixSequence.ps1 and of the controller (UTC). It reads -# files only; Windows PowerShell 5.1 or PowerShell 7. +# The timeline of the cells of the operating-system matrix (Decision 24): for each cell and edition in which the Admin role ran, in the order in which the +# cells ran, the module under test, the account of case 3 (its name and its relative ID, which tells two accounts of one name apart within a domain), +# whether the previous cell had the same name and the same account, when the previous fixture was removed, when the accounts were created, when the +# Admin role started, the minutes between them, and the three effective-access tests of case 3 in the Admin role (result, milliseconds, and the rights +# that a failing test received). A failing effective-access test of the Admin role is easy to blame on the module or on the environment; this table +# puts it beside the module, the position of the cell in the sequence, and the age of the accounts. Pass every label of a series, also a run that +# stopped before its tests (it writes no row, but it created and removed the accounts, which the next cell reports as the previous removal). The +# times come from the logs of Run-MatrixSequence.ps1 and of the controller (UTC). It reads files only; Windows PowerShell 5.1 or PowerShell 7. $ErrorActionPreference = 'Stop' # -File passes an array as one string, so a list may arrive as 'A,B'. $Label = @($Label | ForEach-Object -Process { $_ -split ',' } | Where-Object -FilterScript { $_ }) @@ -36,15 +37,31 @@ foreach ($name in $Label) { $removed = if (Test-Path -LiteralPath $removeLog) { Get-LogTime -Lines @(Get-Content -LiteralPath $removeLog) -Pattern 'Removed the live tests' } $configuration = Get-ChildItem -LiteralPath (Join-Path -Path $folder.FullName -ChildPath 'Results') -Recurse -Filter 'local-*.json' -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -match '-\d{14}\.json$' } | Select-Object -First 1 - $subject = if ($configuration) { (Get-Content -LiteralPath $configuration.FullName -Raw | ConvertFrom-Json).Accounts.Subject } else { $null } + $subjectName = '' + $subjectRid = '' + if ($configuration) { + $subject = (Get-Content -LiteralPath $configuration.FullName -Raw | ConvertFrom-Json).Accounts.Subject + $subjectName = $subject.Name + $subjectRid = ($subject.Sid -split '-')[-1] + } + else { + # A cell that stopped before its tests has no result file, but it created and removed the accounts, so the snapshot of its fixture names them. + $snapshotLog = Join-Path -Path $folder.FullName -ChildPath 'cleanup-1-snapshot.log' + $snapshot = if (Test-Path -LiteralPath $snapshotLog) { Get-Content -LiteralPath $snapshotLog -Raw } + if ($snapshot -match '(?m)^(\w+)\.\S+\s+OU NTFSSecurityLive.*\b(NtfsLiveSubject\w*)=S-[\d-]+-(\d+)') { + $subjectName = '{0}\{1}' -f $Matches[1], $Matches[2] + $subjectRid = $Matches[3] + } + } + $cells.Add([pscustomobject]@{ Run = $name Candidate = $candidate FileServer = $folder.Name.Substring($name.Length + 1) Folder = $folder.FullName Lines = $run - Subject = $(if ($subject) { $subject.Name } else { '' }) - SubjectRid = $(if ($subject) { ($subject.Sid -split '-')[-1] } else { '' }) + Subject = $subjectName + SubjectRid = $subjectRid Started = Get-LogTime -Lines $run -Pattern 'START live tests' Created = Get-LogTime -Lines $run -Pattern 'Preparing the accounts' Removed = $removed @@ -84,6 +101,7 @@ foreach ($cell in ($cells | Sort-Object -Property Started)) { } $hasPrevious = $null -ne $previous -and $null -ne $previous.Removed + $sameName = $null -ne $previous -and $cell.Subject -and $previous.Subject -eq $cell.Subject $rows.Add([pscustomobject][ordered]@{ Run = $cell.Run Candidate = $cell.Candidate @@ -91,7 +109,8 @@ foreach ($cell in ($cells | Sort-Object -Property Started)) { Edition = $edition Subject = $cell.Subject SubjectRid = $cell.SubjectRid - SameSubjectAsPreviousCell = [bool] ($null -ne $previous -and $cell.Subject -and $previous.Subject -eq $cell.Subject) + SameNameAsPreviousCell = [bool] $sameName + SameAccountAsPreviousCell = [bool] ($sameName -and $previous.SubjectRid -eq $cell.SubjectRid) PreviousRemoval = $(if ($hasPrevious) { '{0:yyyy-MM-dd HH:mm:ss}' -f $previous.Removed }) AccountsCreated = '{0:yyyy-MM-dd HH:mm:ss}' -f $cell.Created AdminRoleStarted = '{0:yyyy-MM-dd HH:mm:ss}' -f $adminStart @@ -105,7 +124,7 @@ foreach ($cell in ($cells | Sort-Object -Property Started)) { }) } - $previous = [pscustomobject]@{ Removed = $cell.Removed; Subject = $cell.Subject } + $previous = [pscustomobject]@{ Removed = $cell.Removed; Subject = $cell.Subject; SubjectRid = $cell.SubjectRid } } $rows | Export-Csv -LiteralPath $OutputPath -NoTypeInformation -Encoding ASCII diff --git a/Tests/Lab/Acceptance/Test-MatrixCleanup.ps1 b/Tests/Lab/Acceptance/Test-MatrixCleanup.ps1 index 4ff48d4..844c166 100644 --- a/Tests/Lab/Acceptance/Test-MatrixCleanup.ps1 +++ b/Tests/Lab/Acceptance/Test-MatrixCleanup.ps1 @@ -12,11 +12,12 @@ param ( # records the SIDs of the NtfsLive* accounts while the fixture exists. Verify reads the domains and every machine again, and reports # the organizational unit, the accounts, the share, the folders, the local group, the memberships of Administrators, Access Control # Assistance Operators, and Remote Management Users, and the profiles of those SIDs, and what the suite runs and the probes of the kit leave -# behind (scheduled tasks, items in the stage folders, the folders of the account probe, standard users, probe accounts of the domain). The -# result is judged from this log, never from the wrapper of the controller or a global error count. Repair is for a run whose removal failed: -# with the SIDs of the snapshot, it removes what that run left on the machines (the memberships, also of orphaned SIDs, which net localgroup -# deletes by SID; the share; the local group; the folders) and what the kit leaves (the items in the stage folders, the folders of the -# account probe, the scheduled tasks NtfsMatrix*, and the standard users and domain accounts NtfsProbe*), and then reports like Verify. +# behind (scheduled tasks, items in the stage folders, the folders of the account probe, standard users NtfsProbe* with their profiles and their +# entries in Performance Log Users, probe accounts of the domain). The result is judged from this log, never from the wrapper of the controller +# or a global error count. Repair is for a run whose removal failed: with the SIDs of the snapshot, it removes what that run left on the machines +# (the memberships, also of orphaned SIDs, which net localgroup deletes by SID; the share; the local group; the folders) and what the kit leaves +# (the items in the stage folders, the folders of the account probe, the scheduled tasks NtfsMatrix*, the standard users NtfsProbe* with their +# profiles and their entries in Performance Log Users, and the domain accounts NtfsProbe*), and then reports like Verify. & { $ErrorActionPreference = 'Stop' # -File passes an array as one string, so a list may arrive as 'A,B'. @@ -78,19 +79,29 @@ param ( '{0}: {1} fixture member(s)' -f $groupSid, $hits.Count } + # What the account probe leaves: the profiles and the profile folders of its users, and its entries in Performance Log Users. net.exe lists a + # local user by its bare name, and an entry of a deleted domain account as its SID, or as its name for a while (the cache of names). + $usersFolder = Join-Path -Path $env:SystemDrive -ChildPath 'Users' + $probePaths = @(@(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.LocalPath -like (Join-Path -Path $usersFolder -ChildPath 'NtfsProbe*') } | ForEach-Object -Process { $_.LocalPath }) + + @(Get-ChildItem -LiteralPath $usersFolder -Filter 'NtfsProbe*' -Force -ErrorAction SilentlyContinue | ForEach-Object -Process { $_.FullName }) | Sort-Object -Unique) + $logGroup = ([System.Security.Principal.SecurityIdentifier] 'S-1-5-32-559').Translate([System.Security.Principal.NTAccount]).Value -replace '^.*\\', '' + $probeMembers = @(& net.exe localgroup $logGroup 2>&1 | ForEach-Object -Process { "$_".Trim() } | Where-Object -FilterScript { $_ -match '^S-1-5-21-[\d-]+$' -or $_ -match 'NtfsProbe' }) + [pscustomobject]@{ - Share = [bool] (Get-SmbShare -Name 'NTFSSecurityLive' -ErrorAction SilentlyContinue) - ShareRoot = Test-Path -LiteralPath 'C:\NTFSSecurityLive' - Payload = Test-Path -LiteralPath 'C:\NTFSSecurityLab' - LocalGroup = [bool] (Get-LocalGroup -Name 'NtfsLiveLocal' -ErrorAction SilentlyContinue) - Groups = $groups -join '; ' - Profiles = @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.SID -in $Sid }).Count + Share = [bool] (Get-SmbShare -Name 'NTFSSecurityLive' -ErrorAction SilentlyContinue) + ShareRoot = Test-Path -LiteralPath 'C:\NTFSSecurityLive' + Payload = Test-Path -LiteralPath 'C:\NTFSSecurityLab' + LocalGroup = [bool] (Get-LocalGroup -Name 'NtfsLiveLocal' -ErrorAction SilentlyContinue) + Groups = $groups -join '; ' + Profiles = @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.SID -in $Sid }).Count # What the suite runs and the probes of the kit leave behind: scheduled tasks, items in the stage folders, the folders of the # account probe, and standard users - Tasks = @(Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.TaskName -like 'NtfsMatrix*' }).Count - Stages = @('C:\NtfsMatrixLocal', 'C:\NtfsMatrixProbe' | Where-Object -FilterScript { Test-Path -LiteralPath $_ } | ForEach-Object -Process { Get-ChildItem -LiteralPath $_ -Force -ErrorAction SilentlyContinue }).Count + + Tasks = @(Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.TaskName -like 'NtfsMatrix*' }).Count + Stages = @('C:\NtfsMatrixLocal', 'C:\NtfsMatrixProbe' | Where-Object -FilterScript { Test-Path -LiteralPath $_ } | ForEach-Object -Process { Get-ChildItem -LiteralPath $_ -Force -ErrorAction SilentlyContinue }).Count + @('C:\NtfsProbeRecreation', 'C:\NtfsProbeModules' | Where-Object -FilterScript { Test-Path -LiteralPath $_ }).Count - Users = @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' }).Count + Users = @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' }).Count + ProbeProfiles = $probePaths.Count + ProbeMembers = $probeMembers.Count } } @@ -131,12 +142,28 @@ param ( } Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.TaskName -like 'NtfsMatrix*' } | ForEach-Object -Process { Unregister-ScheduledTask -TaskName $_.TaskName -Confirm:$false -ErrorAction SilentlyContinue } - foreach ($user in @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' })) { - foreach ($userProfile in @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.SID -eq $user.SID.Value })) { Remove-CimInstance -InputObject $userProfile -ErrorAction SilentlyContinue } - Remove-LocalUser -SID $user.SID -ErrorAction SilentlyContinue + foreach ($user in @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' })) { Remove-LocalUser -SID $user.SID -ErrorAction SilentlyContinue } + + # The entries of the probe in Performance Log Users go by SID or name through the cmdlet: net.exe doesn't take the SID of an account that its name cache still resolves. + $logGroup = ([System.Security.Principal.SecurityIdentifier] 'S-1-5-32-559').Translate([System.Security.Principal.NTAccount]).Value -replace '^.*\\', '' + foreach ($member in @(& net.exe localgroup $logGroup 2>&1 | ForEach-Object -Process { "$_".Trim() } | Where-Object -FilterScript { $_ -match '^S-1-5-21-[\d-]+$' -or $_ -match 'NtfsProbe' })) { + Remove-LocalGroupMember -SID 'S-1-5-32-559' -Member $member -ErrorAction SilentlyContinue } - $messages.Add('stage items, probe folders, probe users, and scheduled tasks of the kit removed') + # A profile that the last task of a probe user used stays loaded for a few seconds, so the removal is repeated. What stays is + # reported by the check that follows, found by its folder and not by its user, who is gone by now. + $usersFolder = Join-Path -Path $env:SystemDrive -ChildPath 'Users' + $attempt = 0 + do { + $attempt++ + @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.LocalPath -like (Join-Path -Path $usersFolder -ChildPath 'NtfsProbe*') }) | Remove-CimInstance -ErrorAction SilentlyContinue + Get-ChildItem -LiteralPath $usersFolder -Filter 'NtfsProbe*' -Force -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + $left = @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.LocalPath -like (Join-Path -Path $usersFolder -ChildPath 'NtfsProbe*') }).Count + + @(Get-ChildItem -LiteralPath $usersFolder -Filter 'NtfsProbe*' -Force -ErrorAction SilentlyContinue).Count + if ($left -gt 0 -and $attempt -lt 10) { Start-Sleep -Seconds 3 } + } while ($left -gt 0 -and $attempt -lt 10) + + $messages.Add(('stage items, probe folders, probe users, their entries in the log group, and scheduled tasks of the kit removed; profile items left: {0} after {1} attempt(s)' -f $left, $attempt)) $messages } @@ -149,7 +176,7 @@ param ( $state = Invoke-LabCommand -ComputerName $name -ActivityName "Check $name" -ScriptBlock $machineScript -ArgumentList (, $sids) @labCommand '{0,-9} share={1} C:\NTFSSecurityLive={2} C:\NTFSSecurityLab={3} NtfsLiveLocal={4} profiles={5}' -f $name, $state.Share, $state.ShareRoot, $state.Payload, $state.LocalGroup, $state.Profiles ' {0}' -f $state.Groups - ' residue: scheduled tasks={0} stage items={1} probe users={2}' -f $state.Tasks, $state.Stages, $state.Users + ' residue: scheduled tasks={0} stage items={1} probe users={2} probe profiles={3} probe group members={4}' -f $state.Tasks, $state.Stages, $state.Users, $state.ProbeProfiles, $state.ProbeMembers } } diff --git a/Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1 b/Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1 index 63c101e..fd00bac 100644 --- a/Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1 +++ b/Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1 @@ -28,6 +28,11 @@ param ( # controller that reuses the name would have met a stale entry. -Permutations N asks how often a random assignment of the observed outcomes to the runs # (the same number of failures, a fixed random seed) reaches the best agreement of the real outcomes for some L: if the position of a cell decides the # outcome, it should almost never. +# +# The lifetimes are those of a grid with the step -StepMinutes, so a range is known to within one step (use 0.05 to see the ranges of the record). The model +# doesn't know that a computer restarted. If the state lives in the memory of the computer, a restart would clear it; for the real account names of the series +# of Decision 24, no restart of the client or of a file server changes a prediction (the entry that a restart would have cleared had expired, or the run that +# read it had the same account). For -AsIfSameSubject it matters once: the client restarted between ab6 and ab7. $ErrorActionPreference = 'Stop' $random = New-Object -TypeName 'System.Random' -ArgumentList 20261010 $rows = @(Import-Csv -LiteralPath $Timeline | ForEach-Object -Process { @@ -99,13 +104,44 @@ if ($PSBoundParameters.ContainsKey('Lifetime')) { return } +function Get-Range { + # The lifetimes of a result list as ranges of the grid: 'a to b', or 'a to b and c to d' when the list has a gap. + param ([object[]] $Result) + + $segments = New-Object -TypeName 'System.Collections.Generic.List[string]' + $start = $null + $last = $null + foreach ($item in $Result) { + if ($null -eq $start) { + $start = $item.Minutes + } + elseif ($item.Minutes - $last -gt $StepMinutes * 1.5) { + $segments.Add(('{0:N2} to {1:N2}' -f $start, $last)) + $start = $item.Minutes + } + + $last = $item.Minutes + } + + if ($null -ne $start) { $segments.Add(('{0:N2} to {1:N2}' -f $start, $last)) } + $segments -join ' and ' +} + +# Every lifetime is computed from its index, because adding the step again and again drifts and would lose the last grid point of a range. +$grid = @(for ($step = 0; ; $step++) { + $value = [Math]::Round($FromMinutes + $step * $StepMinutes, 6) + if ($value -gt $ToMinutes) { break } + $value + }) +$resultsByTest = @{} foreach ($test in 'Test1', 'Test2') { - $results = for ($minutes = $FromMinutes; $minutes -le $ToMinutes; $minutes += $StepMinutes) { Test-Model -Minutes $minutes -Test $test } + $results = @(foreach ($minutes in $grid) { Test-Model -Minutes $minutes -Test $test }) + $resultsByTest[$test] = $results $best = ($results | Measure-Object -Property Agree -Maximum).Maximum $bestResults = @($results | Where-Object -FilterScript { $_.Agree -eq $best }) $failures = @($rows | Where-Object -FilterScript { -not $_.$test }).Count - '{0} ({1}): the model predicts {2} of {3} outcomes for L from {4:N2} to {5:N2} minutes; {6} runs failed' -f $test, - $(if ($test -eq 'Test1') { 'the name of the file server' } else { 'the default server name, the client' }), $best, $rows.Count, $bestResults[0].Minutes, $bestResults[-1].Minutes, $failures + '{0} ({1}): the model predicts {2} of {3} outcomes for L from {4} minutes; {5} runs failed' -f $test, + $(if ($test -eq 'Test1') { 'the name of the file server' } else { 'the default server name, the client' }), $best, $rows.Count, (Get-Range -Result $bestResults), $failures if ($ShowMismatches) { foreach ($line in $bestResults[0].Mismatch) { ' mismatch: ' + $line } } if ($Permutations -gt 0) { $observed = [bool[]] @($rows | ForEach-Object -Process { $_.$test }) @@ -127,3 +163,10 @@ foreach ($test in 'Test1', 'Test2') { ' {0} of {1} random assignments of the outcomes to the runs reach {2} of {3} for some L; the best of them reaches {4}' -f $reached, $Permutations, $best, $rows.Count, $highest } } + +# One lifetime for both tests: the range of L at which the model predicts the most outcomes of the two tests together. +$together = @(for ($index = 0; $index -lt $grid.Count; $index++) { + [pscustomobject]@{ Minutes = $grid[$index]; Agree = $resultsByTest['Test1'][$index].Agree + $resultsByTest['Test2'][$index].Agree } + }) +$bestTogether = ($together | Measure-Object -Property Agree -Maximum).Maximum +'Both tests with one L: the model predicts {0} of {1} outcomes for L from {2} minutes' -f $bestTogether, (2 * $rows.Count), (Get-Range -Result @($together | Where-Object -FilterScript { $_.Agree -eq $bestTogether }))