Browse Source

test(lab): count and repair the profiles and log-group entries of the probe, and read the whole series

Test-MatrixCleanup.ps1 now also counts and removes the profiles and the
profile folders C:\Users\NtfsProbe* (retried, because a profile that the
last task used stays loaded for a few seconds) and the entries of the
account probe in Performance Log Users. net.exe lists a local user by its
bare name and a deleted domain account by its SID or its cached name, so
the check matches NtfsProbe anywhere in the line or a SID.

Export-CellTimeline.ps1 takes the account of a cell that stopped before its
tests from the snapshot of its fixture, so that the series can include
such a cell, and reports SameNameAsPreviousCell and SameAccountAsPreviousCell
instead of one column that compared names only.

Test-StaleAuthzModel.ps1 computes its grid of lifetimes by index (the
accumulated step lost the grid point 10.25), prints a range as segments,
and prints the range of one lifetime for both tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: AI Assistant <ai@example.com>
pull/119/head
Raimund Andree 2 days ago
parent
commit
e2384e560b
  1. 43
      Tests/Lab/Acceptance/Export-CellTimeline.ps1
  2. 65
      Tests/Lab/Acceptance/Test-MatrixCleanup.ps1
  3. 49
      Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1

43
Tests/Lab/Acceptance/Export-CellTimeline.ps1

@ -5,13 +5,14 @@ param (
[Parameter(Mandatory)] [string] $OutputPath
)
# The timeline of the cells of the operating-system matrix (Decision 24): for each cell and edition, in the order in which the cells ran, the module under
# test, the account of case 3 (its name and its relative ID, which tells two accounts of one name apart within a domain), when the previous fixture was
# removed, when the accounts were created, when the Admin role started, the minutes between
# them, and the three effective-access tests of case 3 in the Admin role (result, milliseconds, and the rights that a failing test received). A failing
# effective-access test of the Admin role is easy to blame on the module or on the environment; this table puts it beside the module, the position of
# the cell in the sequence, and the age of the accounts. The times come from the logs of Run-MatrixSequence.ps1 and of the controller (UTC). It reads
# files only; Windows PowerShell 5.1 or PowerShell 7.
# The timeline of the cells of the operating-system matrix (Decision 24): for each cell and edition in which the Admin role ran, in the order in which the
# cells ran, the module under test, the account of case 3 (its name and its relative ID, which tells two accounts of one name apart within a domain),
# whether the previous cell had the same name and the same account, when the previous fixture was removed, when the accounts were created, when the
# Admin role started, the minutes between them, and the three effective-access tests of case 3 in the Admin role (result, milliseconds, and the rights
# that a failing test received). A failing effective-access test of the Admin role is easy to blame on the module or on the environment; this table
# puts it beside the module, the position of the cell in the sequence, and the age of the accounts. Pass every label of a series, also a run that
# stopped before its tests (it writes no row, but it created and removed the accounts, which the next cell reports as the previous removal). The
# times come from the logs of Run-MatrixSequence.ps1 and of the controller (UTC). It reads files only; Windows PowerShell 5.1 or PowerShell 7.
$ErrorActionPreference = 'Stop'
# -File passes an array as one string, so a list may arrive as 'A,B'.
$Label = @($Label | ForEach-Object -Process { $_ -split ',' } | Where-Object -FilterScript { $_ })
@ -36,15 +37,31 @@ foreach ($name in $Label) {
$removed = if (Test-Path -LiteralPath $removeLog) { Get-LogTime -Lines @(Get-Content -LiteralPath $removeLog) -Pattern 'Removed the live tests' }
$configuration = Get-ChildItem -LiteralPath (Join-Path -Path $folder.FullName -ChildPath 'Results') -Recurse -Filter 'local-*.json' -ErrorAction SilentlyContinue |
Where-Object -FilterScript { $_.Name -match '-\d{14}\.json$' } | Select-Object -First 1
$subject = if ($configuration) { (Get-Content -LiteralPath $configuration.FullName -Raw | ConvertFrom-Json).Accounts.Subject } else { $null }
$subjectName = ''
$subjectRid = ''
if ($configuration) {
$subject = (Get-Content -LiteralPath $configuration.FullName -Raw | ConvertFrom-Json).Accounts.Subject
$subjectName = $subject.Name
$subjectRid = ($subject.Sid -split '-')[-1]
}
else {
# A cell that stopped before its tests has no result file, but it created and removed the accounts, so the snapshot of its fixture names them.
$snapshotLog = Join-Path -Path $folder.FullName -ChildPath 'cleanup-1-snapshot.log'
$snapshot = if (Test-Path -LiteralPath $snapshotLog) { Get-Content -LiteralPath $snapshotLog -Raw }
if ($snapshot -match '(?m)^(\w+)\.\S+\s+OU NTFSSecurityLive.*\b(NtfsLiveSubject\w*)=S-[\d-]+-(\d+)') {
$subjectName = '{0}\{1}' -f $Matches[1], $Matches[2]
$subjectRid = $Matches[3]
}
}
$cells.Add([pscustomobject]@{
Run = $name
Candidate = $candidate
FileServer = $folder.Name.Substring($name.Length + 1)
Folder = $folder.FullName
Lines = $run
Subject = $(if ($subject) { $subject.Name } else { '' })
SubjectRid = $(if ($subject) { ($subject.Sid -split '-')[-1] } else { '' })
Subject = $subjectName
SubjectRid = $subjectRid
Started = Get-LogTime -Lines $run -Pattern 'START live tests'
Created = Get-LogTime -Lines $run -Pattern 'Preparing the accounts'
Removed = $removed
@ -84,6 +101,7 @@ foreach ($cell in ($cells | Sort-Object -Property Started)) {
}
$hasPrevious = $null -ne $previous -and $null -ne $previous.Removed
$sameName = $null -ne $previous -and $cell.Subject -and $previous.Subject -eq $cell.Subject
$rows.Add([pscustomobject][ordered]@{
Run = $cell.Run
Candidate = $cell.Candidate
@ -91,7 +109,8 @@ foreach ($cell in ($cells | Sort-Object -Property Started)) {
Edition = $edition
Subject = $cell.Subject
SubjectRid = $cell.SubjectRid
SameSubjectAsPreviousCell = [bool] ($null -ne $previous -and $cell.Subject -and $previous.Subject -eq $cell.Subject)
SameNameAsPreviousCell = [bool] $sameName
SameAccountAsPreviousCell = [bool] ($sameName -and $previous.SubjectRid -eq $cell.SubjectRid)
PreviousRemoval = $(if ($hasPrevious) { '{0:yyyy-MM-dd HH:mm:ss}' -f $previous.Removed })
AccountsCreated = '{0:yyyy-MM-dd HH:mm:ss}' -f $cell.Created
AdminRoleStarted = '{0:yyyy-MM-dd HH:mm:ss}' -f $adminStart
@ -105,7 +124,7 @@ foreach ($cell in ($cells | Sort-Object -Property Started)) {
})
}
$previous = [pscustomobject]@{ Removed = $cell.Removed; Subject = $cell.Subject }
$previous = [pscustomobject]@{ Removed = $cell.Removed; Subject = $cell.Subject; SubjectRid = $cell.SubjectRid }
}
$rows | Export-Csv -LiteralPath $OutputPath -NoTypeInformation -Encoding ASCII

65
Tests/Lab/Acceptance/Test-MatrixCleanup.ps1

@ -12,11 +12,12 @@ param (
# records the SIDs of the NtfsLive* accounts while the fixture exists. Verify reads the domains and every machine again, and reports
# the organizational unit, the accounts, the share, the folders, the local group, the memberships of Administrators, Access Control
# Assistance Operators, and Remote Management Users, and the profiles of those SIDs, and what the suite runs and the probes of the kit leave
# behind (scheduled tasks, items in the stage folders, the folders of the account probe, standard users, probe accounts of the domain). The
# result is judged from this log, never from the wrapper of the controller or a global error count. Repair is for a run whose removal failed:
# with the SIDs of the snapshot, it removes what that run left on the machines (the memberships, also of orphaned SIDs, which net localgroup
# deletes by SID; the share; the local group; the folders) and what the kit leaves (the items in the stage folders, the folders of the
# account probe, the scheduled tasks NtfsMatrix*, and the standard users and domain accounts NtfsProbe*), and then reports like Verify.
# behind (scheduled tasks, items in the stage folders, the folders of the account probe, standard users NtfsProbe* with their profiles and their
# entries in Performance Log Users, probe accounts of the domain). The result is judged from this log, never from the wrapper of the controller
# or a global error count. Repair is for a run whose removal failed: with the SIDs of the snapshot, it removes what that run left on the machines
# (the memberships, also of orphaned SIDs, which net localgroup deletes by SID; the share; the local group; the folders) and what the kit leaves
# (the items in the stage folders, the folders of the account probe, the scheduled tasks NtfsMatrix*, the standard users NtfsProbe* with their
# profiles and their entries in Performance Log Users, and the domain accounts NtfsProbe*), and then reports like Verify.
& {
$ErrorActionPreference = 'Stop'
# -File passes an array as one string, so a list may arrive as 'A,B'.
@ -78,19 +79,29 @@ param (
'{0}: {1} fixture member(s)' -f $groupSid, $hits.Count
}
# What the account probe leaves: the profiles and the profile folders of its users, and its entries in Performance Log Users. net.exe lists a
# local user by its bare name, and an entry of a deleted domain account as its SID, or as its name for a while (the cache of names).
$usersFolder = Join-Path -Path $env:SystemDrive -ChildPath 'Users'
$probePaths = @(@(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.LocalPath -like (Join-Path -Path $usersFolder -ChildPath 'NtfsProbe*') } | ForEach-Object -Process { $_.LocalPath }) +
@(Get-ChildItem -LiteralPath $usersFolder -Filter 'NtfsProbe*' -Force -ErrorAction SilentlyContinue | ForEach-Object -Process { $_.FullName }) | Sort-Object -Unique)
$logGroup = ([System.Security.Principal.SecurityIdentifier] 'S-1-5-32-559').Translate([System.Security.Principal.NTAccount]).Value -replace '^.*\\', ''
$probeMembers = @(& net.exe localgroup $logGroup 2>&1 | ForEach-Object -Process { "$_".Trim() } | Where-Object -FilterScript { $_ -match '^S-1-5-21-[\d-]+$' -or $_ -match 'NtfsProbe' })
[pscustomobject]@{
Share = [bool] (Get-SmbShare -Name 'NTFSSecurityLive' -ErrorAction SilentlyContinue)
ShareRoot = Test-Path -LiteralPath 'C:\NTFSSecurityLive'
Payload = Test-Path -LiteralPath 'C:\NTFSSecurityLab'
LocalGroup = [bool] (Get-LocalGroup -Name 'NtfsLiveLocal' -ErrorAction SilentlyContinue)
Groups = $groups -join '; '
Profiles = @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.SID -in $Sid }).Count
Share = [bool] (Get-SmbShare -Name 'NTFSSecurityLive' -ErrorAction SilentlyContinue)
ShareRoot = Test-Path -LiteralPath 'C:\NTFSSecurityLive'
Payload = Test-Path -LiteralPath 'C:\NTFSSecurityLab'
LocalGroup = [bool] (Get-LocalGroup -Name 'NtfsLiveLocal' -ErrorAction SilentlyContinue)
Groups = $groups -join '; '
Profiles = @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.SID -in $Sid }).Count
# What the suite runs and the probes of the kit leave behind: scheduled tasks, items in the stage folders, the folders of the
# account probe, and standard users
Tasks = @(Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.TaskName -like 'NtfsMatrix*' }).Count
Stages = @('C:\NtfsMatrixLocal', 'C:\NtfsMatrixProbe' | Where-Object -FilterScript { Test-Path -LiteralPath $_ } | ForEach-Object -Process { Get-ChildItem -LiteralPath $_ -Force -ErrorAction SilentlyContinue }).Count +
Tasks = @(Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.TaskName -like 'NtfsMatrix*' }).Count
Stages = @('C:\NtfsMatrixLocal', 'C:\NtfsMatrixProbe' | Where-Object -FilterScript { Test-Path -LiteralPath $_ } | ForEach-Object -Process { Get-ChildItem -LiteralPath $_ -Force -ErrorAction SilentlyContinue }).Count +
@('C:\NtfsProbeRecreation', 'C:\NtfsProbeModules' | Where-Object -FilterScript { Test-Path -LiteralPath $_ }).Count
Users = @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' }).Count
Users = @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' }).Count
ProbeProfiles = $probePaths.Count
ProbeMembers = $probeMembers.Count
}
}
@ -131,12 +142,28 @@ param (
}
Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.TaskName -like 'NtfsMatrix*' } | ForEach-Object -Process { Unregister-ScheduledTask -TaskName $_.TaskName -Confirm:$false -ErrorAction SilentlyContinue }
foreach ($user in @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' })) {
foreach ($userProfile in @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.SID -eq $user.SID.Value })) { Remove-CimInstance -InputObject $userProfile -ErrorAction SilentlyContinue }
Remove-LocalUser -SID $user.SID -ErrorAction SilentlyContinue
foreach ($user in @(Get-LocalUser -ErrorAction SilentlyContinue | Where-Object -FilterScript { $_.Name -like 'NtfsProbe*' })) { Remove-LocalUser -SID $user.SID -ErrorAction SilentlyContinue }
# The entries of the probe in Performance Log Users go by SID or name through the cmdlet: net.exe doesn't take the SID of an account that its name cache still resolves.
$logGroup = ([System.Security.Principal.SecurityIdentifier] 'S-1-5-32-559').Translate([System.Security.Principal.NTAccount]).Value -replace '^.*\\', ''
foreach ($member in @(& net.exe localgroup $logGroup 2>&1 | ForEach-Object -Process { "$_".Trim() } | Where-Object -FilterScript { $_ -match '^S-1-5-21-[\d-]+$' -or $_ -match 'NtfsProbe' })) {
Remove-LocalGroupMember -SID 'S-1-5-32-559' -Member $member -ErrorAction SilentlyContinue
}
$messages.Add('stage items, probe folders, probe users, and scheduled tasks of the kit removed')
# A profile that the last task of a probe user used stays loaded for a few seconds, so the removal is repeated. What stays is
# reported by the check that follows, found by its folder and not by its user, who is gone by now.
$usersFolder = Join-Path -Path $env:SystemDrive -ChildPath 'Users'
$attempt = 0
do {
$attempt++
@(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.LocalPath -like (Join-Path -Path $usersFolder -ChildPath 'NtfsProbe*') }) | Remove-CimInstance -ErrorAction SilentlyContinue
Get-ChildItem -LiteralPath $usersFolder -Filter 'NtfsProbe*' -Force -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
$left = @(Get-CimInstance -ClassName Win32_UserProfile | Where-Object -FilterScript { $_.LocalPath -like (Join-Path -Path $usersFolder -ChildPath 'NtfsProbe*') }).Count +
@(Get-ChildItem -LiteralPath $usersFolder -Filter 'NtfsProbe*' -Force -ErrorAction SilentlyContinue).Count
if ($left -gt 0 -and $attempt -lt 10) { Start-Sleep -Seconds 3 }
} while ($left -gt 0 -and $attempt -lt 10)
$messages.Add(('stage items, probe folders, probe users, their entries in the log group, and scheduled tasks of the kit removed; profile items left: {0} after {1} attempt(s)' -f $left, $attempt))
$messages
}
@ -149,7 +176,7 @@ param (
$state = Invoke-LabCommand -ComputerName $name -ActivityName "Check $name" -ScriptBlock $machineScript -ArgumentList (, $sids) @labCommand
'{0,-9} share={1} C:\NTFSSecurityLive={2} C:\NTFSSecurityLab={3} NtfsLiveLocal={4} profiles={5}' -f $name, $state.Share, $state.ShareRoot, $state.Payload, $state.LocalGroup, $state.Profiles
' {0}' -f $state.Groups
' residue: scheduled tasks={0} stage items={1} probe users={2}' -f $state.Tasks, $state.Stages, $state.Users
' residue: scheduled tasks={0} stage items={1} probe users={2} probe profiles={3} probe group members={4}' -f $state.Tasks, $state.Stages, $state.Users, $state.ProbeProfiles, $state.ProbeMembers
}
}

49
Tests/Lab/Acceptance/Test-StaleAuthzModel.ps1

@ -28,6 +28,11 @@ param (
# controller that reuses the name would have met a stale entry. -Permutations N asks how often a random assignment of the observed outcomes to the runs
# (the same number of failures, a fixed random seed) reaches the best agreement of the real outcomes for some L: if the position of a cell decides the
# outcome, it should almost never.
#
# The lifetimes are those of a grid with the step -StepMinutes, so a range is known to within one step (use 0.05 to see the ranges of the record). The model
# doesn't know that a computer restarted. If the state lives in the memory of the computer, a restart would clear it; for the real account names of the series
# of Decision 24, no restart of the client or of a file server changes a prediction (the entry that a restart would have cleared had expired, or the run that
# read it had the same account). For -AsIfSameSubject it matters once: the client restarted between ab6 and ab7.
$ErrorActionPreference = 'Stop'
$random = New-Object -TypeName 'System.Random' -ArgumentList 20261010
$rows = @(Import-Csv -LiteralPath $Timeline | ForEach-Object -Process {
@ -99,13 +104,44 @@ if ($PSBoundParameters.ContainsKey('Lifetime')) {
return
}
function Get-Range {
# The lifetimes of a result list as ranges of the grid: 'a to b', or 'a to b and c to d' when the list has a gap.
param ([object[]] $Result)
$segments = New-Object -TypeName 'System.Collections.Generic.List[string]'
$start = $null
$last = $null
foreach ($item in $Result) {
if ($null -eq $start) {
$start = $item.Minutes
}
elseif ($item.Minutes - $last -gt $StepMinutes * 1.5) {
$segments.Add(('{0:N2} to {1:N2}' -f $start, $last))
$start = $item.Minutes
}
$last = $item.Minutes
}
if ($null -ne $start) { $segments.Add(('{0:N2} to {1:N2}' -f $start, $last)) }
$segments -join ' and '
}
# Every lifetime is computed from its index, because adding the step again and again drifts and would lose the last grid point of a range.
$grid = @(for ($step = 0; ; $step++) {
$value = [Math]::Round($FromMinutes + $step * $StepMinutes, 6)
if ($value -gt $ToMinutes) { break }
$value
})
$resultsByTest = @{}
foreach ($test in 'Test1', 'Test2') {
$results = for ($minutes = $FromMinutes; $minutes -le $ToMinutes; $minutes += $StepMinutes) { Test-Model -Minutes $minutes -Test $test }
$results = @(foreach ($minutes in $grid) { Test-Model -Minutes $minutes -Test $test })
$resultsByTest[$test] = $results
$best = ($results | Measure-Object -Property Agree -Maximum).Maximum
$bestResults = @($results | Where-Object -FilterScript { $_.Agree -eq $best })
$failures = @($rows | Where-Object -FilterScript { -not $_.$test }).Count
'{0} ({1}): the model predicts {2} of {3} outcomes for L from {4:N2} to {5:N2} minutes; {6} runs failed' -f $test,
$(if ($test -eq 'Test1') { 'the name of the file server' } else { 'the default server name, the client' }), $best, $rows.Count, $bestResults[0].Minutes, $bestResults[-1].Minutes, $failures
'{0} ({1}): the model predicts {2} of {3} outcomes for L from {4} minutes; {5} runs failed' -f $test,
$(if ($test -eq 'Test1') { 'the name of the file server' } else { 'the default server name, the client' }), $best, $rows.Count, (Get-Range -Result $bestResults), $failures
if ($ShowMismatches) { foreach ($line in $bestResults[0].Mismatch) { ' mismatch: ' + $line } }
if ($Permutations -gt 0) {
$observed = [bool[]] @($rows | ForEach-Object -Process { $_.$test })
@ -127,3 +163,10 @@ foreach ($test in 'Test1', 'Test2') {
' {0} of {1} random assignments of the outcomes to the runs reach {2} of {3} for some L; the best of them reaches {4}' -f $reached, $Permutations, $best, $rows.Count, $highest
}
}
# One lifetime for both tests: the range of L at which the model predicts the most outcomes of the two tests together.
$together = @(for ($index = 0; $index -lt $grid.Count; $index++) {
[pscustomobject]@{ Minutes = $grid[$index]; Agree = $resultsByTest['Test1'][$index].Agree + $resultsByTest['Test2'][$index].Agree }
})
$bestTogether = ($together | Measure-Object -Property Agree -Maximum).Maximum
'Both tests with one L: the model predicts {0} of {1} outcomes for L from {2} minutes' -f $bestTogether, (2 * $rows.Count), (Get-Range -Result @($together | Where-Object -FilterScript { $_.Agree -eq $bestTogether }))

Loading…
Cancel
Save