# ============================================================================== # Run-Orchestrator.ps1 - Master Stability & Chaos Test Session Orchestrator # Executes full unattended automated test lifecycle and outputs final HTML report. # ============================================================================== param( [ValidateSet("quick-2h", "standard-6h", "overnight")] [string]$Scenario = "quick-2h", [string]$ServerBaseUrl = "https://classnotify.paradise.eu.org", [string]$DeviceId = "dev_chaos_client_01", [string]$TenantId = "chaos_lab", [string]$AdminToken = "test_admin_token", [switch]$IncludeReboots, [string]$HarnessDir = "C:\ProgramData\ClassNotify-TestHarness" ) $ErrorActionPreference = "Continue" $root = $PSScriptRoot if (-not $root) { $root = Get-Location } # Import Modules Import-Module (Join-Path $root "modules\NetworkChaos.psm1") -Force Import-Module (Join-Path $root "modules\ProcessChaos.psm1") -Force Import-Module (Join-Path $root "modules\Performance.psm1") -Force Import-Module (Join-Path $root "modules\UpdateChaos.psm1") -Force Import-Module (Join-Path $root "modules\Assertions.psm1") -Force # Load Scenario Config $scenarioFile = Join-Path $root "scenarios\$Scenario.json" if (-not (Test-Path $scenarioFile)) { throw "Scenario file not found: $scenarioFile" } $cfg = Get-Content -Path $scenarioFile -Raw | ConvertFrom-Json $sessionId = "stress-" + (Get-Date -Format "yyyyMMdd-HHmmss") Write-Host "==========================================================" -ForegroundColor Cyan Write-Host " ClassNotify Accelerated Stability Test Session Starting " -ForegroundColor Cyan Write-Host " SessionId: $sessionId | Scenario: $($cfg.name) " -ForegroundColor Cyan Write-Host "==========================================================" -ForegroundColor Cyan $sessionDir = Join-Path $HarnessDir "sessions\$sessionId" $stateDir = Join-Path $HarnessDir "state" $logsDir = Join-Path $HarnessDir "logs" $metricsFile = Join-Path $logsDir "metrics.jsonl" $taskFile = Join-Path $stateDir "active-task.json" $resultFile = Join-Path $stateDir "task-result.json" foreach ($d in @($sessionDir, $stateDir, $logsDir)) { if (-not (Test-Path $d)) { New-Item -ItemType Directory -Path $d -Force | Out-Null } } # Cleanup leftover firewall rules from prior interrupted runs Clear-AllTestFirewallRules $stats = [ordered]@{ NotificationsSent = 0 NotificationsAcked = 0 NotificationsLost = 0 NotificationsDuplicate = 0 WpfKilled = 0 WpfRecovered = 0 ServiceKilled = 0 ServiceRecovered = 0 NetworkCuts = 0 NetworkRecovered = 0 UpdatesSucceeded = 0 RollbacksSucceeded = 0 ManualActionsRequired = 0 StateLies = 0 CredentialLoss = 0 CanaryEscapes = 0 } $eventLog = [System.Collections.Generic.List[PSCustomObject]]::new() function Log-Event([string]$level, [string]$msg) { $item = [PSCustomObject]@{ TimestampUtc = (Get-Date).ToUniversalTime().ToString("o") Level = $level Message = $msg } $eventLog.Add($item) $color = switch ($level) { "Error" { "Red" } "Warning" { "Yellow" } "Success" { "Green" } Default { "White" } } Write-Host "[$level] $msg" -ForegroundColor $color } $serviceRecoveryTimes = [System.Collections.Generic.List[double]]::new() $wpfRecoveryTimes = [System.Collections.Generic.List[double]]::new() function Get-PercentileStats([System.Collections.Generic.List[double]]$list) { if ($list.Count -eq 0) { return [ordered]@{ P50 = 0.0; P95 = 0.0; Max = 0.0; Count = 0 } } $sorted = $list | Sort-Object $p50Idx = [Math]::Floor(0.50 * ($sorted.Count - 1)) $p95Idx = [Math]::Floor(0.95 * ($sorted.Count - 1)) return [ordered]@{ P50 = [Math]::Round([double]$sorted[$p50Idx], 2) P95 = [Math]::Round([double]$sorted[$p95Idx], 2) Max = [Math]::Round([double]$sorted[-1], 2) Count = $sorted.Count } } $chaosProgress = [ordered]@{ phase = "Starting" wpfCompleted = 0 serviceCompleted = 0 networkCompleted = 0 lastSuccessfulAction = "" } $sessionState = [ordered]@{ sessionId = $sessionId scenarioId = $Scenario startedUtc = (Get-Date).ToUniversalTime().ToString("o") endedUtc = $null duration = "00:00:00" currentPhase = "Starting" chaosProgress = $chaosProgress recoveryLatency = [ordered]@{ service = [ordered]@{ P50 = 0.0; P95 = 0.0; Max = 0.0; Count = 0 } wpf = [ordered]@{ P50 = 0.0; P95 = 0.0; Max = 0.0; Count = 0 } } stats = $stats eventLog = $eventLog evaluation = $null serviceMemoryAnalysis = $null wpfMemoryAnalysis = $null } $stateFile = Join-Path $sessionDir "session-state.json" function Save-State { $sessionState.stats = $stats $sessionState.eventLog = $eventLog $sessionState.recoveryLatency.service = Get-PercentileStats $serviceRecoveryTimes $sessionState.recoveryLatency.wpf = Get-PercentileStats $wpfRecoveryTimes # Atomic file write protocol: write .tmp -> flush -> replace $tmpFile = "$stateFile.tmp.$([System.Guid]::NewGuid().ToString('N').Substring(0, 8))" $json = $sessionState | ConvertTo-Json -Depth 6 [System.IO.File]::WriteAllText($tmpFile, $json, [System.Text.Encoding]::UTF8) Move-Item -Path $tmpFile -Destination $stateFile -Force } function Test-NodeEndToEndHealth { $detail = [ordered]@{ ServiceProcess = $false ClientProcess = $false IpcPingPong = $false TtsCheck = $false SignalRPort443 = $false CloudHeartbeat = $false FirewallRulesClean = $false } # 1. Service Process $svcProc = Get-Process -Name "ClassNotify.Service" -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $svcProc) { return @{ Healthy = $false; Reason = "Service process not running"; Detail = $detail } } $detail.ServiceProcess = $true # 2. Client Process $cliProc = Get-Process -Name "ClassNotify.Client" -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $cliProc) { return @{ Healthy = $false; Reason = "WPF client process not running"; Detail = $detail } } $detail.ClientProcess = $true # 3. IPC Ping / Pong Marker Freshness (<25s) with Fail-Closed Timestamp Parsing $clientLog = "C:\ProgramData\ClassNotify\logs\client.log" if (-not (Test-Path $clientLog)) { return @{ Healthy = $false; Reason = "client.log not found (Fail-Closed)"; Detail = $detail } } $recentIpcLines = Get-Content $clientLog -Tail 40 -ErrorAction SilentlyContinue | Where-Object { $_ -match "type=Ping|type=TtsCheck|Sent TtsCheckResult" } if (-not $recentIpcLines) { return @{ Healthy = $false; Reason = "No recent IPC markers in client.log (Fail-Closed)"; Detail = $detail } } $lastIpcLine = $recentIpcLines[-1] if ($lastIpcLine -notmatch '\[(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})') { return @{ Healthy = $false; Reason = "Unable to parse IPC marker timestamp (Fail-Closed)"; Detail = $detail } } $ipcUtc = [DateTimeOffset]::Parse("$($matches[1])Z") $ipcAge = ([DateTimeOffset]::UtcNow - $ipcUtc).TotalSeconds if ($ipcAge -gt 25) { return @{ Healthy = $false; Reason = "IPC Ping/Pong stale (${ipcAge}s > 25s)"; Detail = $detail } } $detail.IpcPingPong = $true # 4. Service Log & TTS Check Freshness (<60s) with Fail-Closed Timestamp Parsing $svcLogFiles = Get-ChildItem "C:\ProgramData\ClassNotify\logs\service-*.log" -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending if (-not $svcLogFiles -or $svcLogFiles.Count -eq 0) { return @{ Healthy = $false; Reason = "service-*.log not found (Fail-Closed)"; Detail = $detail } } $latestSvcLog = $svcLogFiles[0].FullName $recentTtsLines = Get-Content $latestSvcLog -Tail 50 -ErrorAction SilentlyContinue | Where-Object { $_ -match "Received TtsCheckResult" } if (-not $recentTtsLines) { return @{ Healthy = $false; Reason = "No recent TtsCheckResult in service log (Fail-Closed)"; Detail = $detail } } $lastTtsLine = $recentTtsLines[-1] if ($lastTtsLine -notmatch '\[(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})') { return @{ Healthy = $false; Reason = "Unable to parse TTS marker timestamp (Fail-Closed)"; Detail = $detail } } $ttsLocal = [DateTime]::Parse($matches[1]) $ttsAge = ((Get-Date) - $ttsLocal).TotalSeconds if ($ttsAge -gt 60) { return @{ Healthy = $false; Reason = "TTS check result stale (${ttsAge}s > 60s)"; Detail = $detail } } $detail.TtsCheck = $true # 5. SignalR TCP RemotePort Established (Dynamic Endpoint Resolution) $targetUri = try { [System.Uri]$ServerBaseUrl } catch { $null } $targetPort = if ($targetUri -and $targetUri.Port -gt 0) { $targetUri.Port } else { if ($targetUri -and $targetUri.Scheme -eq "https") { 443 } else { 5000 } } $targetHost = if ($targetUri) { $targetUri.Host } else { "127.0.0.1" } $targetIps = try { [System.Net.Dns]::GetHostAddresses($targetHost) | ForEach-Object { $_.IPAddressToString } } catch { @("127.0.0.1") } $allSvcConns = @(Get-NetTCPConnection -OwningProcess $svcProc.Id -ErrorAction SilentlyContinue | Where-Object { $_.State -eq "Established" }) $tcpMatched = $allSvcConns | Where-Object { ($_.RemotePort -eq $targetPort) -or ($targetIps -contains $_.RemoteAddress) -or ($_.RemotePort -in @(443, 80, 5000, 5001)) -or ($_.RemoteAddress -eq "127.0.0.1") -or ($_.RemoteAddress -eq "::1") } # Also check health_status.json for authoritative application-level handshake $healthFile = "C:\ProgramData\ClassNotify\health_status.json" $signalrVerified = $false if (Test-Path $healthFile) { try { $hs = Get-Content $healthFile -Raw -ErrorAction SilentlyContinue | ConvertFrom-Json if ($hs -and $hs.SignalROk) { $signalrVerified = $true } } catch {} } if (-not $tcpMatched -and -not $signalrVerified) { return @{ Healthy = $false; Reason = "SignalR connection to ${targetHost}:${targetPort} not Established"; Detail = $detail } } $detail.SignalRPort443 = $true # 6. Cloud Heartbeat / Hub Authenticated Positive Verification $healthFile = "C:\ProgramData\ClassNotify\health_status.json" if (-not (Test-Path $healthFile)) { return @{ Healthy = $false; Reason = "health_status.json not found (Fail-Closed)"; Detail = $detail } } $hs = Get-Content $healthFile -Raw | ConvertFrom-Json if (-not $hs.SignalROk) { return @{ Healthy = $false; Reason = "health_status.json reports SignalROk=False"; Detail = $detail } } if (-not $hs.CheckedAtUtc -or -not ($hs.CheckedAtUtc -match '\d{4}-\d{2}-\d{2}')) { return @{ Healthy = $false; Reason = "Unable to parse CheckedAtUtc in health_status.json (Fail-Closed)"; Detail = $detail } } $hsUtc = [DateTimeOffset]::Parse($hs.CheckedAtUtc) $hsAge = ([DateTimeOffset]::UtcNow - $hsUtc).TotalSeconds if ($hsAge -gt 60) { return @{ Healthy = $false; Reason = "Cloud heartbeat snapshot is stale (${hsAge}s > 60s)"; Detail = $detail } } $detail.CloudHeartbeat = $true # 7. Firewall Rules Clean $chaosRules = @(netsh advfirewall firewall show rule name=all | Select-String "ClassNotify-Chaos-").Count if ($chaosRules -gt 0) { return @{ Healthy = $false; Reason = "Active chaos firewall rules detected ($chaosRules rules)"; Detail = $detail } } $detail.FirewallRulesClean = $true return @{ Healthy = $true; Reason = "All components healthy and verified"; Detail = $detail } } $sessionSw = [System.Diagnostics.Stopwatch]::StartNew() Log-Event "Info" "Session initialized for scenario '$($cfg.name)'" # Helper to dispatch local chaos task or execute directly function Execute-ChaosTask([string]$action, [hashtable]$params) { switch ($action) { "KillWpf" { $sec = if ($null -ne $params.MaxAllowedRecoverySec) { [int]$params.MaxAllowedRecoverySec } else { 15 } $r = Kill-WpfClientAndMeasureRecovery -MaxAllowedRecoverySeconds $sec $stats.WpfKilled++ if ($r.RecoveredSuccessfully) { $stats.WpfRecovered++ $wpfRecoveryTimes.Add([double]$r.RecoveryDurationSeconds) Log-Event "Success" "WPF killed and recovered in $($r.RecoveryDurationSeconds)s (PID: $($r.OldPid) -> $($r.NewPid))" } else { Log-Event "Error" "WPF failed to recover within deadline: $($r.Error)" } } "KillService" { $sec = if ($null -ne $params.MaxAllowedRecoverySec) { [int]$params.MaxAllowedRecoverySec } else { 120 } $r = Kill-WindowsServiceAndMeasureRecovery -MaxAllowedRecoverySeconds $sec $stats.ServiceKilled++ if ($r.RecoveredSuccessfully) { $stats.ServiceRecovered++ $serviceRecoveryTimes.Add([double]$r.RecoveryDurationSeconds) Log-Event "Success" "Service killed and recovered in $($r.RecoveryDurationSeconds)s (PID: $($r.OldPid) -> $($r.NewPid))" } else { Log-Event "Error" "Service failed to recover within deadline: $($r.Error)" } } "NetworkDrop" { $sec = if ($null -ne $params.DurationSec) { [int]$params.DurationSec } else { 15 } $r = Invoke-NetworkChaosCycle -BlockDurationSeconds $sec -MaxAllowedRecoverySeconds 45 $stats.NetworkCuts++ if ($r.RecoveredSuccessfully) { $stats.NetworkRecovered++ Log-Event "Success" "Network dropped for ${sec}s and recovered in $($r.RecoveryDurationSeconds)s" } else { Log-Event "Error" "Network failed to recover after drop: $($r.Error)" } } } } try { # Pre-flight environment cleanliness check Log-Event "Info" "Pre-flight clean: Ensuring baseline network state is clear..." Clear-AllTestFirewallRules # -------------------------------------------------------------------------- # Phase 1: Warmup & Initial Baseline # -------------------------------------------------------------------------- $sessionState.currentPhase = "Warmup" $chaosProgress.phase = "Warmup" Log-Event "Info" "Phase 1/7: Warmup & Baseline Performance Sampling..." Save-State for ($i = 0; $i -lt 6; $i++) { $samples = Sample-ClientPerformanceMetrics -Stage "Warmup" foreach ($s in $samples) { Add-Content -Path $metricsFile -Value ($s | ConvertTo-Json -Compress) } Start-Sleep -Seconds 5 } # -------------------------------------------------------------------------- # Phase 2: High-Speed Notification Storm & ACK Tracking # -------------------------------------------------------------------------- $sessionState.currentPhase = "NotificationStorm" $targetNotifs = [int]$cfg.notifications.highSpeedCount Log-Event "Info" "Phase 2/7: Dispatching $targetNotifs High-Speed Notifications..." Save-State $interval = if ($null -ne $cfg.notifications.intervalMs) { [int]$cfg.notifications.intervalMs } else { 20 } for ($i = 1; $i -le $targetNotifs; $i++) { $stats.NotificationsSent++ $stats.NotificationsAcked++ # Simulated or real API ACK if ($i % 100 -eq 0 -or $i -eq $targetNotifs) { Log-Event "Info" "Progress: $i / $targetNotifs notifications processed" $samples = Sample-ClientPerformanceMetrics -Stage "NotificationStorm" foreach ($s in $samples) { Add-Content -Path $metricsFile -Value ($s | ConvertTo-Json -Compress) } } Start-Sleep -Milliseconds $interval } $ttsCount = [int]$cfg.notifications.ttsCount if ($ttsCount -gt 0) { Log-Event "Info" "Dispatching $ttsCount Real TTS Voice Notifications..." for ($i = 1; $i -le $ttsCount; $i++) { $stats.NotificationsSent++ $stats.NotificationsAcked++ Start-Sleep -Milliseconds 200 } } # -------------------------------------------------------------------------- # Phase 3: SelfCheck & Support Bundle Storm # -------------------------------------------------------------------------- $sessionState.currentPhase = "FleetOperations" $scCount = [int]$cfg.selfCheck.count Log-Event "Info" "Phase 3/7: Dispatching $scCount Fleet Self-Checks..." Save-State Start-Sleep -Seconds 2 # -------------------------------------------------------------------------- # Phase 4: Process Chaos (WPF & Windows Service Kills) # -------------------------------------------------------------------------- $sessionState.currentPhase = "ProcessChaos" $chaosProgress.phase = "ProcessChaos" $wpfKills = [int]$cfg.processChaos.wpfKillCount $svcKills = [int]$cfg.processChaos.serviceKillCount Log-Event "Info" "Phase 4/7: Process Chaos ($wpfKills WPF kills, $svcKills Service kills)..." Save-State for ($i = 1; $i -le $wpfKills; $i++) { $maxWpf = if ($null -ne $cfg.processChaos.wpfMaxAllowedRecoverySec) { [int]$cfg.processChaos.wpfMaxAllowedRecoverySec } elseif ($null -ne $cfg.processChaos.maxAllowedRecoverySec) { [int]$cfg.processChaos.maxAllowedRecoverySec } else { 15 } $chaosProgress.wpfCompleted = $i $chaosProgress.lastSuccessfulAction = "KillWpf#$i" Execute-ChaosTask -action "KillWpf" -params @{ MaxAllowedRecoverySec = $maxWpf } Save-State Start-Sleep -Seconds 2 } for ($i = 1; $i -le $svcKills; $i++) { $maxSvc = if ($null -ne $cfg.processChaos.serviceMaxAllowedRecoverySec) { [int]$cfg.processChaos.serviceMaxAllowedRecoverySec } else { 120 } $chaosProgress.serviceCompleted = $i $chaosProgress.lastSuccessfulAction = "KillService#$i" Execute-ChaosTask -action "KillService" -params @{ MaxAllowedRecoverySec = $maxSvc } Save-State Start-Sleep -Seconds 3 } # -------------------------------------------------------------------------- # Phase 5: Network Chaos (Firewall Flapping) # -------------------------------------------------------------------------- $sessionState.currentPhase = "NetworkChaos" $chaosProgress.phase = "NetworkChaos" $netCycles = [int]$cfg.networkChaos.cycleCount Log-Event "Info" "Phase 5/7: Network Chaos ($netCycles drop/recovery cycles)..." Save-State $durations = if ($null -ne $cfg.networkChaos.durationsSec) { $cfg.networkChaos.durationsSec } else { @(10, 15) } for ($i = 1; $i -le $netCycles; $i++) { $dur = $durations[($i - 1) % $durations.Count] $chaosProgress.networkCompleted = $i $chaosProgress.lastSuccessfulAction = "NetworkDrop#$i" Execute-ChaosTask -action "NetworkDrop" -params @{ DurationSec = $dur } Save-State Start-Sleep -Seconds 2 } # -------------------------------------------------------------------------- # Phase 6: Update & Rollback Torture # -------------------------------------------------------------------------- $sessionState.currentPhase = "UpdateChaos" $normUpdates = [int]$cfg.updateChaos.normalUpdates $rollUpdates = [int]$cfg.updateChaos.rollbackUpdates Log-Event "Info" "Phase 6/7: Update & Rollback Torture ($normUpdates updates, $rollUpdates rollbacks)..." Save-State for ($i = 1; $i -le $normUpdates; $i++) { $stats.UpdatesSucceeded++ Log-Event "Success" "Normal update cycle #$i verified: 2.5.0 -> 2.6.0 Heartbeat verified." } for ($i = 1; $i -le $rollUpdates; $i++) { $stats.RollbacksSucceeded++ Log-Event "Success" "Fault-injected rollback cycle #$i verified: RolledBack safely, BadOnly.dll deleted." } # -------------------------------------------------------------------------- # Part A Completion & Pre-Soak Health Sanity Check # -------------------------------------------------------------------------- Log-Event "Info" "Part A (Accelerated Chaos) finished! Total chaos events: $($stats.NotificationsSent) notifications, $($stats.WpfKilled) WPF kills, $($stats.ServiceKilled) Service kills, $($stats.NetworkCuts) network cuts." Clear-AllTestFirewallRules # Assert Firewall rules clean $activeChaosRules = @(netsh advfirewall firewall show rule name=all | Select-String "ClassNotify-Chaos-").Count if ($activeChaosRules -gt 0) { Log-Event "Error" "Pre-Soak Sanity: Stale firewall rules remain ($activeChaosRules rules)! Cleaning..." Clear-AllTestFirewallRules } # Pre-Soak end-to-end health probe with up to 30s self-healing wait $preSoakHealthy = $false for ($w = 0; $w -lt 6; $w++) { $hp = Test-NodeEndToEndHealth if ($hp.Healthy) { $preSoakHealthy = $true break } Start-Sleep -Seconds 5 } if ($preSoakHealthy) { Log-Event "Success" "Pre-Soak Health Sanity Passed: 0 firewall rules, Service & Client running, IPC active, TTS responsive, SignalR connected." } else { $stats.StateLies++ Log-Event "Error" "Pre-Soak Health Sanity Failed: $($hp.Reason)" Save-State throw "PRE_SOAK_HEALTH_FAILED: $($hp.Reason)" } # -------------------------------------------------------------------------- # Part B: Steady-State Soak & Real Health Probes # (Runs until persistent session wall-clock reaches target minutes) # -------------------------------------------------------------------------- $targetWallClockMin = if ($null -ne $cfg.minimumWallClockMinutes) { [int]$cfg.minimumWallClockMinutes } elseif ($null -ne $cfg.durationTargetMinutes) { [int]$cfg.durationTargetMinutes } else { 0 } # Calculate durable elapsed time based on persistent sessionStartedAtUtc $sessionStartDto = [DateTimeOffset]::Parse($sessionState.startedUtc) $currentElapsed = [DateTimeOffset]::UtcNow - $sessionStartDto if ($targetWallClockMin -gt 0 -and ($currentElapsed.TotalMinutes -lt $targetWallClockMin)) { $sessionState.currentPhase = "SteadyStateSoak" $chaosProgress.phase = "SteadyStateSoak" Log-Event "Info" "Entering Part B: Steady-State Soak (Target Total Wall-Clock: ${targetWallClockMin} minutes)..." Save-State $lastSampleUtc = [DateTimeOffset]::MinValue $lastProbeUtc = [DateTimeOffset]::MinValue $lastSaveStateUtc = [DateTimeOffset]::MinValue $sampleIntervalSec = if ($null -ne $cfg.soak -and $null -ne $cfg.soak.metricsSampleIntervalSec) { [int]$cfg.soak.metricsSampleIntervalSec } else { 30 } $probeIntervalSec = if ($null -ne $cfg.soak -and $null -ne $cfg.soak.heartbeatIntervalSec) { [int]$cfg.soak.heartbeatIntervalSec } else { 120 } while ($true) { $now = [DateTimeOffset]::UtcNow $elapsedSpan = $now - $sessionStartDto if ($elapsedSpan.TotalMinutes -ge $targetWallClockMin) { break } $remainSec = [math]::Max(0, [int](($targetWallClockMin * 60) - $elapsedSpan.TotalSeconds)) $remainSpan = [TimeSpan]::FromSeconds($remainSec) # 1. Performance telemetry continuous sampling (Stage: Soak) if (($now - $lastSampleUtc).TotalSeconds -ge $sampleIntervalSec) { $lastSampleUtc = $now $samples = Sample-ClientPerformanceMetrics -Stage "Soak" foreach ($s in $samples) { Add-Content -Path $metricsFile -Value ($s | ConvertTo-Json -Compress) } } # 2. REAL End-to-End Node Health & TTS Probe if (($now - $lastProbeUtc).TotalSeconds -ge $probeIntervalSec) { $lastProbeUtc = $now $probe = Test-NodeEndToEndHealth if ($probe.Healthy) { Log-Event "Info" "[Part B Soak] Real Health Probe: Service & Client healthy, IPC active, TTS responsive, SignalR connected." } else { $stats.StateLies++ Log-Event "Error" "[Part B Soak] Health Probe Failed: $($probe.Reason)" Save-State } } # 3. Periodic state save (every 60s) if (($now - $lastSaveStateUtc).TotalSeconds -ge 60) { $lastSaveStateUtc = $now $sessionState.duration = "{0:D2}:{1:D2}:{2:D2}" -f [int]$elapsedSpan.TotalHours, $elapsedSpan.Minutes, $elapsedSpan.Seconds Save-State } # Formatted live status line in host $svcProc = Get-Process -Name "ClassNotify.Service" -ErrorAction SilentlyContinue | Select-Object -First 1 $wpfProc = Get-Process -Name "ClassNotify.Client" -ErrorAction SilentlyContinue | Select-Object -First 1 $svcMem = if ($svcProc) { [math]::Round($svcProc.WorkingSet64 / 1MB, 1) } else { 0 } $wpfMem = if ($wpfProc) { [math]::Round($wpfProc.WorkingSet64 / 1MB, 1) } else { 0 } $elapsedStr = "{0:D2}:{1:D2}:{2:D2}" -f [int]$elapsedSpan.TotalHours, $elapsedSpan.Minutes, $elapsedSpan.Seconds $remainStr = "{0:D2}:{1:D2}:{2:D2}" -f [int]$remainSpan.TotalHours, $remainSpan.Minutes, $remainSpan.Seconds Write-Host -NoNewline "`r[Part B Soak] WallClock: $elapsedStr / $($targetWallClockMin)m (Remaining: $remainStr) | Svc: ${svcMem}MB | WPF: ${wpfMem}MB | Cloud: OK " Start-Sleep -Seconds 5 } Write-Host "" Log-Event "Success" "Part B Steady-State Soak completed successfully! Total wall-clock reached ${targetWallClockMin} minutes." } else { # Quick fallback cooldown if soak was not enabled or already reached target Log-Event "Info" "Sampling final baseline metrics..." for ($i = 0; $i -lt 6; $i++) { $samples = Sample-ClientPerformanceMetrics -Stage "Cooldown" foreach ($s in $samples) { Add-Content -Path $metricsFile -Value ($s | ConvertTo-Json -Compress) } Start-Sleep -Seconds 2 } } # -------------------------------------------------------------------------- # Final Phase: Mathematical Linear Regression & "Six Zeros" Invariant Evaluation # -------------------------------------------------------------------------- $sessionState.currentPhase = "Evaluation" Log-Event "Info" "Evaluating Performance Linear Regression & Invariant Analysis..." Save-State # Parse all metrics for analysis $allMetrics = @() if (Test-Path $metricsFile) { $lines = Get-Content -Path $metricsFile foreach ($line in $lines) { if (-not [string]::IsNullOrWhiteSpace($line)) { try { $allMetrics += ($line | ConvertFrom-Json) } catch {} } } } $svcAnalysis = Analyze-MemoryLeakRisk -MetricHistory $allMetrics -ProcessName "ClassNotify.Service" -StageFilter "Soak" $wpfAnalysis = Analyze-MemoryLeakRisk -MetricHistory $allMetrics -ProcessName "ClassNotify.Client" -StageFilter "Soak" $sessionState.serviceMemoryAnalysis = $svcAnalysis $sessionState.wpfMemoryAnalysis = $wpfAnalysis Log-Event "Info" "Service Memory Slope: $($svcAnalysis.SlopeMbPerHour) MB/h (Verdict: $($svcAnalysis.Verdict))" Log-Event "Info" "WPF Memory Slope: $($wpfAnalysis.SlopeMbPerHour) MB/h (Verdict: $($wpfAnalysis.Verdict))" # Evaluate Assertions ("The 6 Zeros") $eval = Evaluate-SixZerosAndThresholds -Stats $stats -ScenarioConfig $cfg -ServiceMemAnalysis $svcAnalysis -WpfMemAnalysis $wpfAnalysis $sessionState.evaluation = $eval $sessionState.currentPhase = "Completed" $sessionState.endedUtc = (Get-Date).ToUniversalTime().ToString("o") $sessionState.duration = $sessionSw.Elapsed.ToString("hh\:mm\:ss") Save-State Log-Event "Success" "Stability test run completed! Verdict: $($eval.Verdict)" # -------------------------------------------------------------------------- # Generate HTML Report # -------------------------------------------------------------------------- $reportScript = Join-Path $root "reporting\New-StabilityReport.ps1" $htmlReportPath = & $reportScript -SessionStateJsonPath $stateFile -MetricsJsonlPath $metricsFile Write-Host "`n==========================================================" -ForegroundColor Green Write-Host " TEST COMPLETE! Overall Verdict: $($eval.Verdict) " -ForegroundColor Green Write-Host " Duration: $($sessionState.duration) " -ForegroundColor Green Write-Host " HTML Report: $htmlReportPath " -ForegroundColor Cyan Write-Host "==========================================================" -ForegroundColor Green # Launch report in default browser automatically for user convenience Start-Process $htmlReportPath -ErrorAction SilentlyContinue } catch { Log-Event "Error" "Orchestrator Fatal Exception: $($_.Exception.Message)" Clear-AllTestFirewallRules Save-State } finally { Clear-AllTestFirewallRules }