using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IdentityModel.Tokens.Jwt; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Security.Claims; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using ClassNotify.Server.Configuration; using ClassNotify.Server.Controllers; using ClassNotify.Server.Data; using ClassNotify.Server.Domain.Entities; using ClassNotify.Server.Services; using ClassNotify.Shared.Contracts; using ClassNotify.Shared.Models; using ClassNotify.Shared.Services; using System.Security.Cryptography; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http.Connections; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.SignalR.Client; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.IdentityModel.Tokens; using Npgsql; namespace ClassNotify.CommandTests; public class CustomWebApplicationFactory : WebApplicationFactory { public const string TestJwtKey = "THIS_IS_A_VERY_LONG_TEST_SIGNING_KEY_EXCEEDING_SIXTY_FOUR_BYTES_IN_LENGTH_FOR_COMMAND_TESTS!"; public const string TestIssuer = "ClassNotify.Server"; public const string TestAudience = "ClassNotify.Clients"; private readonly string _testDbName; public CustomWebApplicationFactory(string? dbName = null) { _testDbName = dbName ?? ("classnotify_command_test_" + Guid.NewGuid().ToString("N")); } public string DbName => _testDbName; protected override void ConfigureWebHost(IWebHostBuilder builder) { var dir = new DirectoryInfo(AppContext.BaseDirectory); while (dir != null && !File.Exists(Path.Combine(dir.FullName, "ClassNotify.sln"))) { dir = dir.Parent; } var rootDir = dir?.FullName ?? Directory.GetCurrentDirectory(); var serverPath = Path.Combine(rootDir, "src", "Server", "ClassNotify.Server"); builder.UseContentRoot(serverPath); builder.UseEnvironment("Development"); builder.ConfigureAppConfiguration((context, config) => { config.AddInMemoryCollection(new Dictionary { ["RateLimit:AuthPermitLimit"] = "200", ["RateLimit:ActivationPermitLimit"] = "200", ["RateLimit:SendPermitLimit"] = "200", ["Jwt:SigningKey"] = TestJwtKey, ["Jwt:Issuer"] = TestIssuer, ["Jwt:Audience"] = TestAudience, ["Jwt:LifetimeDays"] = "14", ["ConnectionStrings:DefaultConnection"] = "Host=127.0.0.1;Database=classnotify_test;Username=postgres;" }); }); builder.ConfigureServices(services => { var dbDescriptors = services.Where(d => d.ServiceType == typeof(DbContextOptions) || d.ServiceType == typeof(ClassNotifyDbContext)).ToList(); foreach (var d in dbDescriptors) { services.Remove(d); } services.AddDbContext(options => { options.UseInMemoryDatabase(_testDbName) .ConfigureWarnings(w => w.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)); }); }); } public static string CreateJwt(string userId, string userName, string role, string? tenantId) { var tokenHandler = new JwtSecurityTokenHandler(); var key = Encoding.UTF8.GetBytes(TestJwtKey); var claims = new List { new(ClaimTypes.NameIdentifier, userId), new(ClaimTypes.Name, userName), new(ClaimTypes.Role, role), new("token_version", "1") }; if (!string.IsNullOrEmpty(tenantId)) { claims.Add(new Claim("tenant_id", tenantId)); } var tokenDescriptor = new SecurityTokenDescriptor { Subject = new ClaimsIdentity(claims), Issuer = TestIssuer, Audience = TestAudience, Expires = DateTime.UtcNow.AddDays(7), SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key), SecurityAlgorithms.HmacSha256Signature) }; var token = tokenHandler.CreateToken(tokenDescriptor); return tokenHandler.WriteToken(token); } } public class Program { private static int _passCount = 0; private static int _failCount = 0; public static async Task Main(string[] args) { Console.WriteLine("=========================================================="); Console.WriteLine(" ClassNotify P1 Batch 10: Remote Command Test Suite "); Console.WriteLine("=========================================================="); await RunTestAsync("Scenario 1: Whitelist Strict Check & Reject Arbitrary Commands", Test_01_Whitelist_Strict_Check_Reject_Arbitrary_Commands); await RunTestAsync("Scenario 2: Target Device Validation & Multi-Tenant Isolation", Test_02_Target_Device_Validation_And_Tenant_Isolation); await RunTestAsync("Scenario 3: Online Dispatch via SignalR Transitions Status to Dispatched", Test_03_Online_Dispatch_Via_SignalR_Transitions_To_Dispatched); await RunTestAsync("Scenario 4: Offline Queuing Preserves Pending Status With Short TTL", Test_04_Offline_Queuing_Preserves_Pending_Status); await RunTestAsync("Scenario 5: Reconnect Replay Within TTL (Unicast to Caller Device)", Test_05_Reconnect_Replay_Within_TTL_Unicast_To_Caller); await RunTestAsync("Scenario 6: Short-Lived TTL Expiration Never Replays Past ExpiresAt", Test_06_Short_Lived_TTL_Expiration_Never_Replays_Past_ExpiresAt); await RunTestAsync("Scenario 7: DeviceCommandDeduplicator Thread Safety (Concurrent Duplicates)", Test_07_Device_Command_Deduplicator_Prevents_Concurrent_Duplicate_Execution); await RunTestAsync("Scenario 8: Device-Side Idempotency Cache Returns Previous Execution Result", Test_08_Device_Side_Idempotency_Cache_Returns_Previous_Result_Without_ReExecution); await RunTestAsync("Scenario 9: Execution Result Reporting Updates DB Status & Timings", Test_09_Result_Reporting_Updates_DB_Status_And_Timings); await RunTestAsync("Scenario 10: Stale & Spoofed Result Rejection (Cross-Device Isolation)", Test_10_Stale_And_Spoofed_Result_Rejection_Device_Isolation); await RunTestAsync("Scenario 11: TeacherHub Relay On Command Status Changed", Test_11_Teacher_Hub_Relay_On_Command_Status_Changed); await RunTestAsync("Scenario 12: Command Audit Trail History Query & Tenant Filters", Test_12_Command_Audit_Trail_History_Query_And_Filters); await RunTestAsync("Scenario 13: PostgreSQL Schema & Migration Integrity", Test_13_PostgreSQL_Schema_And_Migration_Integrity); await RunTestAsync("Scenario 14: Timeout Worker Automatically Transitions Expired & TimedOut Commands", Test_14_Timeout_Worker_Automatically_Transitions_Expired_And_TimedOut); await RunTestAsync("Scenario 15: Durable Command Ledger Survives Restart & Retains Deduplication", Test_15_Durable_Command_Ledger_Survives_Restart_And_Retains_Deduplication); await RunTestAsync("Scenario 16: Command DurationMs Entity Persistence & Audit Query", Test_16_Command_DurationMs_Entity_Persistence_And_Audit_Query); await RunTestAsync("Scenario 17: Fail-Closed Ledger Write Failure Blocks Execution", Test_17_Fail_Closed_Ledger_Write_Failure_Blocks_Execution); await RunTestAsync("Scenario 18: Fail-Closed Corrupted Ledger Rejects Commands & Prevents Re-Execution", Test_18_Fail_Closed_Corrupted_Ledger_On_Restart_Rejects_Commands); Console.WriteLine(); Console.WriteLine("=========================================================="); Console.WriteLine($" Test Results: {_passCount} Passed, {_failCount} Failed"); Console.WriteLine("=========================================================="); return _failCount > 0 ? 1 : 0; } private static async Task RunTestAsync(string name, Func testAction) { try { await testAction(); _passCount++; Console.WriteLine($"[PASS] {name}"); } catch (Exception ex) { _failCount++; Console.WriteLine($"[FAIL] {name}"); Console.WriteLine($" Exception: {ex.Message}"); Console.WriteLine($" Stack: {ex.StackTrace}"); } } private static async Task SeedBaselineAsync(CustomWebApplicationFactory factory) { using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); if (!await db.Tenants.AnyAsync(t => t.Id == "t_alpha")) { db.Tenants.AddRange( new Tenant { Id = "t_alpha", Name = "Alpha School", IsActive = true, CreatedAt = DateTimeOffset.UtcNow }, new Tenant { Id = "t_beta", Name = "Beta School", IsActive = true, CreatedAt = DateTimeOffset.UtcNow } ); db.Classrooms.AddRange( new Classroom { Id = "cls_alpha_1", TenantId = "t_alpha", DisplayName = "高一(1)班", Grade = "10", RoomNumber = "101", CreatedAt = DateTimeOffset.UtcNow }, new Classroom { Id = "cls_beta_1", TenantId = "t_beta", DisplayName = "初一(1)班", Grade = "7", RoomNumber = "701", CreatedAt = DateTimeOffset.UtcNow } ); db.Devices.AddRange( new Device { Id = "dev_alpha_1", TenantId = "t_alpha", ClassroomId = "cls_alpha_1", DeviceTokenHash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes("tok_alpha_1_secret"))), Hostname = "ALPHA-PC-1", IsActive = true, CreatedAt = DateTimeOffset.UtcNow }, new Device { Id = "dev_beta_1", TenantId = "t_beta", ClassroomId = "cls_beta_1", DeviceTokenHash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes("tok_beta_1_secret"))), Hostname = "BETA-PC-1", IsActive = true, CreatedAt = DateTimeOffset.UtcNow } ); db.Users.AddRange( new User { Id = "u_teacher_a", Username = "teacher_a", PasswordHash = "hash", FullName = "张老师", Role = "Teacher", TenantId = "t_alpha", IsLocked = false, TokenVersion = 1, CreatedAt = DateTimeOffset.UtcNow }, new User { Id = "u_teacher_b", Username = "teacher_b", PasswordHash = "hash", FullName = "王老师", Role = "Teacher", TenantId = "t_beta", IsLocked = false, TokenVersion = 1, CreatedAt = DateTimeOffset.UtcNow }, new User { Id = "u_super", Username = "super", PasswordHash = "hash", FullName = "SuperAdmin", Role = "SuperAdmin", TenantId = "", IsLocked = false, TokenVersion = 1, CreatedAt = DateTimeOffset.UtcNow } ); await db.SaveChangesAsync(); } } #region Scenario Tests private static async Task Test_01_Whitelist_Strict_Check_Reject_Arbitrary_Commands() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); var teacherToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherToken); // 1. Whitelist Commands: SelfCheck, RestartClient, Reconnect, RefreshConfig MUST succeed var whitelistTypes = new[] { "SelfCheck", "RestartClient", "Reconnect", "RefreshConfig" }; foreach (var cmdType in whitelistTypes) { var req = new SendDeviceCommandRequest { DeviceId = "dev_alpha_1", CommandType = cmdType }; var resp = await client.PostAsJsonAsync("/api/device-command/send", req); if (!resp.IsSuccessStatusCode) { var body = await resp.Content.ReadAsStringAsync(); throw new Exception($"Whitelist command '{cmdType}' was rejected: Status={resp.StatusCode}, Body={body}"); } var json = await resp.Content.ReadFromJsonAsync(); if (!json.GetProperty("success").GetBoolean()) throw new Exception($"Expected success=true for command '{cmdType}'"); var cmd = json.GetProperty("command"); if (cmd.GetProperty("commandType").GetString() != cmdType) throw new Exception($"Returned commandType does not match: expected {cmdType}"); } // 2. Arbitrary / Forbidden Commands MUST be strictly rejected with 400 Bad Request var forbiddenTypes = new[] { "powershell -enc aW13...", "cmd /c whoami", "shutdown /s /t 0", "rmdir /s /q C:\\", "bash -c rm -rf /", "eval('require(\"child_process\")')", "calc.exe", "" }; foreach (var badCmd in forbiddenTypes) { var req = new SendDeviceCommandRequest { DeviceId = "dev_alpha_1", CommandType = badCmd }; var resp = await client.PostAsJsonAsync("/api/device-command/send", req); if (resp.StatusCode != HttpStatusCode.BadRequest) { throw new Exception($"Forbidden command '{badCmd}' was not rejected with 400! Got {resp.StatusCode}"); } } } private static async Task Test_02_Target_Device_Validation_And_Tenant_Isolation() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); // 1. Non-existent device -> 404 Not Found var superToken = CustomWebApplicationFactory.CreateJwt("u_super", "SuperAdmin", "SuperAdmin", null); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var notFoundReq = new SendDeviceCommandRequest { DeviceId = "dev_non_existent_999", CommandType = "SelfCheck" }; var resp404 = await client.PostAsJsonAsync("/api/device-command/send", notFoundReq); if (resp404.StatusCode != HttpStatusCode.NotFound) { throw new Exception($"Expected 404 for non-existent device, got {resp404.StatusCode}"); } // 2. Multi-tenant isolation: Teacher of Alpha cannot send command to Beta device -> 403 Forbidden var teacherAlphaToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherAlphaToken); var crossTenantReq = new SendDeviceCommandRequest { DeviceId = "dev_beta_1", // Belongs to t_beta! CommandType = "SelfCheck" }; var resp403 = await client.PostAsJsonAsync("/api/device-command/send", crossTenantReq); if (resp403.StatusCode != HttpStatusCode.Forbidden) { throw new Exception($"Cross-tenant command was not rejected with 403! Got {resp403.StatusCode}"); } // 3. SuperAdmin CAN manage devices across all tenants -> 200 OK client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var respSuper = await client.PostAsJsonAsync("/api/device-command/send", crossTenantReq); if (!respSuper.IsSuccessStatusCode) { throw new Exception($"SuperAdmin cross-tenant dispatch failed: {respSuper.StatusCode}"); } } private static async Task Test_03_Online_Dispatch_Via_SignalR_Transitions_To_Dispatched() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); // 1. Connect Device Alpha via SignalR var receivedCommands = new ConcurrentBag(); var deviceHubUrl = "http://localhost/hubs/device"; var hubConn = new HubConnectionBuilder() .WithUrl(deviceHubUrl, options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); hubConn.On("ReceiveRemoteCommand", cmd => { receivedCommands.Add(cmd); }); await hubConn.StartAsync(); // Wait until DeviceHub OnConnectedAsync finishes registering the connection using (var scope = factory.Services.CreateScope()) { var sec = scope.ServiceProvider.GetRequiredService(); var waitConn = 0; while (string.IsNullOrEmpty(sec.GetDeviceConnectionId("dev_alpha_1")) && waitConn < 40) { await Task.Delay(50); waitConn++; } } try { // 2. Send command via API var teacherToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherToken); var req = new SendDeviceCommandRequest { DeviceId = "dev_alpha_1", CommandType = "SelfCheck" }; var resp = await client.PostAsJsonAsync("/api/device-command/send", req); if (!resp.IsSuccessStatusCode) throw new Exception($"Send command failed: {resp.StatusCode}"); var json = await resp.Content.ReadFromJsonAsync(); if (!json.GetProperty("isDispatched").GetBoolean()) throw new Exception("Expected isDispatched=true for online device"); var cmdDto = json.GetProperty("command"); var commandId = cmdDto.GetProperty("commandId").GetString(); // 3. Verify SignalR client received command within 2 seconds var spinWait = 0; while (receivedCommands.IsEmpty && spinWait < 40) { await Task.Delay(50); spinWait++; } if (receivedCommands.IsEmpty) throw new Exception("Online device did not receive command via SignalR in time"); var received = receivedCommands.First(); if (received.CommandId != commandId) throw new Exception($"Received CommandId mismatch: expected {commandId}, got {received.CommandId}"); if (received.CommandType != "SelfCheck") throw new Exception($"Received CommandType mismatch: expected SelfCheck, got {received.CommandType}"); // 4. Verify DB status is Dispatched using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var dbCmd = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == commandId); if (dbCmd == null || dbCmd.Status != DeviceCommandStatuses.Dispatched) throw new Exception($"Expected DB status 'Dispatched', got '{dbCmd?.Status}'"); } finally { await hubConn.StopAsync(); } } private static async Task Test_04_Offline_Queuing_Preserves_Pending_Status() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); // Device dev_alpha_1 is currently offline var teacherToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherToken); var req = new SendDeviceCommandRequest { DeviceId = "dev_alpha_1", CommandType = "RestartClient" }; var resp = await client.PostAsJsonAsync("/api/device-command/send", req); if (!resp.IsSuccessStatusCode) throw new Exception($"Offline send failed: {resp.StatusCode}"); var json = await resp.Content.ReadFromJsonAsync(); if (json.GetProperty("isDispatched").GetBoolean()) throw new Exception("Expected isDispatched=false for offline device"); var cmdDto = json.GetProperty("command"); var commandId = cmdDto.GetProperty("commandId").GetString(); var status = cmdDto.GetProperty("status").GetString(); if (status != DeviceCommandStatuses.Pending) throw new Exception($"Expected Pending status, got {status}"); // Verify DB record and short TTL (RestartClient has 2 min TTL) using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var dbCmd = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == commandId); if (dbCmd == null) throw new Exception("Command record not found in DB"); if (dbCmd.Status != DeviceCommandStatuses.Pending) throw new Exception($"DB status is {dbCmd.Status}"); var ttlRemaining = dbCmd.ExpiresAt - DateTimeOffset.UtcNow; if (ttlRemaining.TotalSeconds < 30 || ttlRemaining.TotalMinutes > 3) throw new Exception($"Unexpected TTL duration: {ttlRemaining.TotalSeconds}s (expected ~2 min)"); } private static async Task Test_05_Reconnect_Replay_Within_TTL_Unicast_To_Caller() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); // 1. Create a pending command for dev_alpha_1 while offline var teacherToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherToken); var req = new SendDeviceCommandRequest { DeviceId = "dev_alpha_1", CommandType = "RefreshConfig" }; var resp = await client.PostAsJsonAsync("/api/device-command/send", req); var json = await resp.Content.ReadFromJsonAsync(); var expectedCmdId = json.GetProperty("command").GetProperty("commandId").GetString(); // 2. Connect another device (dev_beta_1) first var betaReceived = new ConcurrentBag(); var betaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_beta_1"; options.Headers["X-Device-Token"] = "tok_beta_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); betaConn.On("ReceiveRemoteCommand", cmd => betaReceived.Add(cmd)); await betaConn.StartAsync(); // 3. Now connect target device (dev_alpha_1) var alphaReceived = new ConcurrentBag(); var alphaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); alphaConn.On("ReceiveRemoteCommand", cmd => alphaReceived.Add(cmd)); await alphaConn.StartAsync(); try { // Wait up to 2 seconds for replay var spin = 0; while (alphaReceived.IsEmpty && spin < 40) { await Task.Delay(50); spin++; } if (alphaReceived.IsEmpty) throw new Exception("Reconnecting device did not receive queued pending command"); var replayed = alphaReceived.First(); if (replayed.CommandId != expectedCmdId) throw new Exception($"Replayed CommandId mismatch: expected {expectedCmdId}, got {replayed.CommandId}"); if (replayed.CommandType != "RefreshConfig") throw new Exception($"Replayed CommandType mismatch: expected RefreshConfig, got {replayed.CommandType}"); // Unicast check: dev_beta_1 MUST have received NOTHING if (!betaReceived.IsEmpty) throw new Exception("Unicast isolation breach! Another device received the command replayed on reconnect"); // Verify status in DB transitioned to Dispatched using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var dbCmd = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == expectedCmdId); if (dbCmd?.Status != DeviceCommandStatuses.Dispatched) throw new Exception($"Expected DB status Dispatched after reconnect replay, got {dbCmd?.Status}"); } finally { await alphaConn.StopAsync(); await betaConn.StopAsync(); } } private static async Task Test_06_Short_Lived_TTL_Expiration_Never_Replays_Past_ExpiresAt() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); // 1. Manually insert an expired pending command into DB var expiredCmdId = $"cmd_exp_{Guid.NewGuid():N}"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.DeviceCommands.Add(new DeviceCommand { Id = expiredCmdId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "RestartClient", CreatedBy = "Admin", CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-10), ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(-5), // Expired 5 minutes ago! Status = DeviceCommandStatuses.Pending }); await db.SaveChangesAsync(); } // 2. Connect device var received = new ConcurrentBag(); var hubConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); hubConn.On("ReceiveRemoteCommand", cmd => received.Add(cmd)); await hubConn.StartAsync(); try { await Task.Delay(1000); // Expired command must NEVER be delivered! if (!received.IsEmpty) throw new Exception($"Safety violation: expired command was delivered on reconnect: {received.First().CommandId}"); // Verify status transitioned to Expired in DB using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var cmd = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == expiredCmdId); if (cmd?.Status != DeviceCommandStatuses.Expired) throw new Exception($"Expected DB status 'Expired', got '{cmd?.Status}'"); } } finally { await hubConn.StopAsync(); } } private static async Task Test_07_Device_Command_Deduplicator_Prevents_Concurrent_Duplicate_Execution() { var deduplicator = new DeviceCommandDeduplicator(maxCapacity: 100); var testCmdId = $"cmd_concurrent_{Guid.NewGuid():N}"; int winCount = 0; int rejectCount = 0; var tasks = Enumerable.Range(0, 30).Select(_ => Task.Run(() => { if (deduplicator.TryBeginExecution(testCmdId)) { Interlocked.Increment(ref winCount); } else { Interlocked.Increment(ref rejectCount); } })).ToArray(); await Task.WhenAll(tasks); if (winCount != 1) throw new Exception($"Expected exactly 1 execution permit for command {testCmdId}, but got {winCount}!"); if (rejectCount != 29) throw new Exception($"Expected 29 rejected permits, but got {rejectCount}"); } private static async Task Test_08_Device_Side_Idempotency_Cache_Returns_Previous_Result_Without_ReExecution() { var deduplicator = new DeviceCommandDeduplicator(maxCapacity: 100); var cmdId = $"cmd_cache_test_{Guid.NewGuid():N}"; // First attempt begins execution if (!deduplicator.TryBeginExecution(cmdId)) throw new Exception("First TryBeginExecution must return true"); // While in-flight, TryGetCachedResult returns false if (deduplicator.TryGetCachedResult(cmdId, out _)) throw new Exception("TryGetCachedResult must return false while command is in-flight"); // Complete execution with result var expectedResult = new DeviceCommandResultDto { CommandId = cmdId, DeviceId = "dev_alpha_1", Status = DeviceCommandStatuses.Succeeded, ResultCode = 0, ResultMessage = "SelfCheck diagnostics completed cleanly", ExecutionDurationMs = 125, Timestamp = DateTimeOffset.UtcNow }; deduplicator.CompleteExecution(cmdId, expectedResult); // Next TryBeginExecution must return false (already executed) if (deduplicator.TryBeginExecution(cmdId)) throw new Exception("TryBeginExecution after CompleteExecution must return false"); // TryGetCachedResult must return the previously cached outcome if (!deduplicator.TryGetCachedResult(cmdId, out var cachedResult) || cachedResult == null) throw new Exception("TryGetCachedResult must return true after CompleteExecution"); if (cachedResult.CommandId != cmdId || cachedResult.Status != DeviceCommandStatuses.Succeeded || cachedResult.ExecutionDurationMs != 125) throw new Exception("Cached result content mismatch!"); await Task.CompletedTask; } private static async Task Test_09_Result_Reporting_Updates_DB_Status_And_Timings() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var cmdId = $"cmd_report_{Guid.NewGuid():N}"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.DeviceCommands.Add(new DeviceCommand { Id = cmdId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "SelfCheck", CreatedBy = "Teacher", CreatedAt = DateTimeOffset.UtcNow, ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(5), Status = DeviceCommandStatuses.Dispatched }); await db.SaveChangesAsync(); } // Connect device and call ReportCommandResult var hubConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); await hubConn.StartAsync(); try { var reportDto = new DeviceCommandResultDto { CommandId = cmdId, DeviceId = "dev_alpha_1", Status = DeviceCommandStatuses.Succeeded, ResultCode = 0, ResultMessage = "SelfCheck passed: 4/4 subsystems healthy", ExecutionDurationMs = 240, Timestamp = DateTimeOffset.UtcNow }; await hubConn.InvokeAsync("ReportCommandResult", reportDto); await Task.Delay(500); using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var updated = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == cmdId); if (updated?.Status != DeviceCommandStatuses.Succeeded) throw new Exception($"Expected DB status 'Succeeded', got '{updated?.Status}'"); if (updated.ResultCode != 0) throw new Exception($"Expected ResultCode 0, got {updated.ResultCode}"); if (updated.CompletedAt == null) throw new Exception("CompletedAt timestamp was not recorded"); if (!updated.ResultMessage!.Contains("4/4 subsystems healthy")) throw new Exception($"ResultMessage mismatch: {updated.ResultMessage}"); } finally { await hubConn.StopAsync(); } } private static async Task Test_10_Stale_And_Spoofed_Result_Rejection_Device_Isolation() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); // Command belongs to dev_beta_1 in t_beta var betaCmdId = $"cmd_beta_{Guid.NewGuid():N}"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.DeviceCommands.Add(new DeviceCommand { Id = betaCmdId, TenantId = "t_beta", DeviceId = "dev_beta_1", CommandType = "RestartClient", CreatedBy = "BetaAdmin", CreatedAt = DateTimeOffset.UtcNow, ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(2), Status = DeviceCommandStatuses.Dispatched }); await db.SaveChangesAsync(); } // Connect as dev_alpha_1 in t_alpha, and attempt to report result for betaCmdId var alphaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); await alphaConn.StartAsync(); try { var spoofedResult = new DeviceCommandResultDto { CommandId = betaCmdId, DeviceId = "dev_beta_1", // Spoofed device ID Status = DeviceCommandStatuses.Succeeded, ResultCode = 0, ResultMessage = "Hacked result from dev_alpha_1", Timestamp = DateTimeOffset.UtcNow }; await alphaConn.InvokeAsync("ReportCommandResult", spoofedResult); await Task.Delay(500); // Verify betaCmdId status was NOT modified by dev_alpha_1! using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var targetCmd = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == betaCmdId); if (targetCmd?.Status != DeviceCommandStatuses.Dispatched) throw new Exception($"Device isolation breach! Spoofed result altered command status to '{targetCmd?.Status}'"); if (targetCmd.ResultMessage?.Contains("Hacked") == true) throw new Exception("Device isolation breach! Spoofed message was written into target command"); } finally { await alphaConn.StopAsync(); } } private static async Task Test_11_Teacher_Hub_Relay_On_Command_Status_Changed() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var cmdId = $"cmd_relay_{Guid.NewGuid():N}"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.DeviceCommands.Add(new DeviceCommand { Id = cmdId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "SelfCheck", CreatedBy = "TeacherAlpha", CreatedAt = DateTimeOffset.UtcNow, ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(5), Status = DeviceCommandStatuses.Dispatched }); await db.SaveChangesAsync(); } // 1. Connect Teacher Alpha to TeacherHub var teacherAlphaToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); var alphaEvents = new ConcurrentBag(); var teacherAlphaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/teacher", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["Authorization"] = $"Bearer {teacherAlphaToken}"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); teacherAlphaConn.On("OnDeviceCommandStatusChanged", res => alphaEvents.Add(res)); await teacherAlphaConn.StartAsync(); // 2. Connect Teacher Beta to TeacherHub (different tenant) var teacherBetaToken = CustomWebApplicationFactory.CreateJwt("u_teacher_b", "王老师", "Teacher", "t_beta"); var betaEvents = new ConcurrentBag(); var teacherBetaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/teacher", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["Authorization"] = $"Bearer {teacherBetaToken}"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); teacherBetaConn.On("OnDeviceCommandStatusChanged", res => betaEvents.Add(res)); await teacherBetaConn.StartAsync(); // 3. Device Alpha reports command result var deviceAlphaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); await deviceAlphaConn.StartAsync(); try { var report = new DeviceCommandResultDto { CommandId = cmdId, DeviceId = "dev_alpha_1", Status = DeviceCommandStatuses.Succeeded, ResultCode = 0, ResultMessage = "SelfCheck completed cleanly", ExecutionDurationMs = 150, Timestamp = DateTimeOffset.UtcNow }; await deviceAlphaConn.InvokeAsync("ReportCommandResult", report); var spin = 0; while (alphaEvents.IsEmpty && spin < 40) { await Task.Delay(50); spin++; } if (alphaEvents.IsEmpty) throw new Exception("Teacher Alpha did not receive OnDeviceCommandStatusChanged event in time"); var relayed = alphaEvents.First(); if (relayed.CommandId != cmdId || relayed.Status != DeviceCommandStatuses.Succeeded) throw new Exception($"Relayed event content mismatch: {relayed.CommandId}, {relayed.Status}"); // Tenant isolation: Teacher Beta must have received NOTHING if (!betaEvents.IsEmpty) throw new Exception("Tenant isolation violation! Teacher Beta received command event belonging to Tenant Alpha"); } finally { await deviceAlphaConn.StopAsync(); await teacherAlphaConn.StopAsync(); await teacherBetaConn.StopAsync(); } } private static async Task Test_12_Command_Audit_Trail_History_Query_And_Filters() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); // Populate history records using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var now = DateTimeOffset.UtcNow; db.DeviceCommands.AddRange( new DeviceCommand { Id = "hist_a1", TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "SelfCheck", CreatedBy = "Admin", CreatedAt = now.AddMinutes(-30), ExpiresAt = now.AddMinutes(-25), Status = DeviceCommandStatuses.Succeeded, ResultCode = 0 }, new DeviceCommand { Id = "hist_a2", TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "RestartClient", CreatedBy = "Admin", CreatedAt = now.AddMinutes(-20), ExpiresAt = now.AddMinutes(-18), Status = DeviceCommandStatuses.Failed, ResultCode = -10 }, new DeviceCommand { Id = "hist_b1", TenantId = "t_beta", DeviceId = "dev_beta_1", CommandType = "SelfCheck", CreatedBy = "Admin", CreatedAt = now.AddMinutes(-10), ExpiresAt = now.AddMinutes(-5), Status = DeviceCommandStatuses.Succeeded, ResultCode = 0 } ); await db.SaveChangesAsync(); } // 1. Teacher Alpha queries history: must see ONLY t_alpha commands var teacherAlphaToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherAlphaToken); var respAlpha = await client.GetFromJsonAsync>("/api/device-command/history"); if (respAlpha == null || respAlpha.Count == 0) throw new Exception("Teacher Alpha received empty history"); if (respAlpha.Any(c => c.TenantId != "t_alpha")) throw new Exception("Teacher Alpha history contains records from another tenant!"); // 2. Filter by status=Failed var respFailed = await client.GetFromJsonAsync>("/api/device-command/history?status=Failed"); if (respFailed == null || respFailed.Count != 1 || respFailed[0].CommandId != "hist_a2") throw new Exception("Status filter Failed did not return expected record hist_a2"); // 3. SuperAdmin queries history: sees both tenants var superToken = CustomWebApplicationFactory.CreateJwt("u_super", "SuperAdmin", "SuperAdmin", null); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var respSuper = await client.GetFromJsonAsync>("/api/device-command/history"); if (respSuper == null || !respSuper.Any(c => c.TenantId == "t_alpha") || !respSuper.Any(c => c.TenantId == "t_beta")) throw new Exception("SuperAdmin history does not contain multi-tenant records"); } private static async Task Test_13_PostgreSQL_Schema_And_Migration_Integrity() { // 1. Verify EF Core Migration code structure var migration = new ClassNotify.Server.Data.Migrations.AddDeviceCommandsTable(); var operationsProperty = typeof(Migration).GetProperty("UpOperations", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); var operations = operationsProperty?.GetValue(migration) as IReadOnlyList; if (operations == null || operations.Count == 0) { var buildUpMethod = typeof(ClassNotify.Server.Data.Migrations.AddDeviceCommandsTable) .GetMethod("Up", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); var mb = new MigrationBuilder("Npgsql"); buildUpMethod?.Invoke(migration, new object[] { mb }); operations = mb.Operations; } var createTableOp = operations.OfType() .FirstOrDefault(t => t.Name == "DeviceCommands"); if (createTableOp == null) throw new Exception("Migration 20260922120000_AddDeviceCommandsTable lacks CreateTableOperation for DeviceCommands"); var colNames = createTableOp.Columns.Select(c => c.Name).ToHashSet(); var requiredCols = new[] { "Id", "TenantId", "DeviceId", "CommandType", "Parameters", "CreatedBy", "CreatedAt", "ExpiresAt", "Status", "ResultCode", "ResultMessage" }; foreach (var reqCol in requiredCols) { if (!colNames.Contains(reqCol)) throw new Exception($"DeviceCommands column missing in migration: {reqCol}"); } // Check indexes var createIndexOps = operations.OfType().ToList(); if (!createIndexOps.Any(i => i.Name == "IX_DeviceCommands_DeviceId_Status")) throw new Exception("Migration lacks IX_DeviceCommands_DeviceId_Status index"); if (!createIndexOps.Any(i => i.Name == "IX_DeviceCommands_Status_ExpiresAt")) throw new Exception("Migration lacks IX_DeviceCommands_Status_ExpiresAt index"); if (!createIndexOps.Any(i => i.Name == "IX_DeviceCommands_TenantId_DeviceId_CreatedAt")) throw new Exception("Migration lacks IX_DeviceCommands_TenantId_DeviceId_CreatedAt index"); // 2. Real PostgreSQL Migration Integration Test if connection string is configured var pgConnStr = Environment.GetEnvironmentVariable("CLASSNOTIFY_TEST_DB_CONNECTION"); if (!string.IsNullOrWhiteSpace(pgConnStr)) { Console.WriteLine(" Executing live PostgreSQL migration test on Oracle VPS..."); var baseBuilder = new NpgsqlConnectionStringBuilder(pgConnStr); string tempDbName = $"cn_cmd_test_{Guid.NewGuid():N}"; var tempBuilder = new NpgsqlConnectionStringBuilder(pgConnStr) { Database = tempDbName }; // Create temporary DB await using (var conn = new NpgsqlConnection(baseBuilder.ConnectionString)) { await conn.OpenAsync(); await using var cmd = new NpgsqlCommand($"CREATE DATABASE \"{tempDbName}\";", conn); await cmd.ExecuteNonQueryAsync(); } try { var opts = new DbContextOptionsBuilder(); opts.UseNpgsql(tempBuilder.ConnectionString); using (var db = new ClassNotifyDbContext(opts.Options)) { // Apply migrations up to Batch 10 await db.Database.MigrateAsync(); // Verify table and indexes via information_schema await using var checkCmd = db.Database.GetDbConnection().CreateCommand(); if (checkCmd.Connection!.State != System.Data.ConnectionState.Open) await checkCmd.Connection.OpenAsync(); checkCmd.CommandText = "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'DeviceCommands';"; var tableCount = Convert.ToInt32(await checkCmd.ExecuteScalarAsync()); if (tableCount != 1) throw new Exception("PostgreSQL table 'DeviceCommands' does not exist after MigrateAsync!"); checkCmd.CommandText = "SELECT indexname FROM pg_indexes WHERE tablename = 'DeviceCommands';"; var indexes = new List(); await using (var reader = await checkCmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { indexes.Add(reader.GetString(0)); } } if (!indexes.Contains("IX_DeviceCommands_DeviceId_Status")) throw new Exception("PostgreSQL index IX_DeviceCommands_DeviceId_Status missing in real database"); if (!indexes.Contains("IX_DeviceCommands_Status_ExpiresAt")) throw new Exception("PostgreSQL index IX_DeviceCommands_Status_ExpiresAt missing in real database"); if (!indexes.Contains("IX_DeviceCommands_TenantId_DeviceId_CreatedAt")) throw new Exception("PostgreSQL index IX_DeviceCommands_TenantId_DeviceId_CreatedAt missing in real database"); checkCmd.CommandText = "SELECT column_name FROM information_schema.columns WHERE table_name = 'DeviceCommands';"; var columns = new List(); await using (var colReader = await checkCmd.ExecuteReaderAsync()) { while (await colReader.ReadAsync()) { columns.Add(colReader.GetString(0)); } } if (!columns.Contains("DurationMs")) throw new Exception("PostgreSQL column 'DurationMs' missing in DeviceCommands table"); } } finally { // Clean up temporary database NpgsqlConnection.ClearAllPools(); await using var dropConn = new NpgsqlConnection(baseBuilder.ConnectionString); await dropConn.OpenAsync(); await using var dropCmd = new NpgsqlCommand($"DROP DATABASE IF EXISTS \"{tempDbName}\" WITH (FORCE);", dropConn); await dropCmd.ExecuteNonQueryAsync(); } } else { Console.WriteLine(" [INFO] CLASSNOTIFY_TEST_DB_CONNECTION not set; live PostgreSQL test skipped."); } } private static async Task Test_14_Timeout_Worker_Automatically_Transitions_Expired_And_TimedOut() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var now = DateTimeOffset.UtcNow; var pendingExpiredId = $"cmd_pend_exp_{Guid.NewGuid():N}"; var dispatchedExpiredId = $"cmd_disp_exp_{Guid.NewGuid():N}"; var dispatchedTimedOutId = $"cmd_disp_to_{Guid.NewGuid():N}"; var dispatchedFreshId = $"cmd_disp_fresh_{Guid.NewGuid():N}"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); // 1. Pending command past ExpiresAt db.DeviceCommands.Add(new DeviceCommand { Id = pendingExpiredId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "SelfCheck", CreatedBy = "Teacher", CreatedAt = now.AddMinutes(-10), ExpiresAt = now.AddMinutes(-5), Status = DeviceCommandStatuses.Pending }); // 2. Dispatched command past ExpiresAt db.DeviceCommands.Add(new DeviceCommand { Id = dispatchedExpiredId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "RestartClient", CreatedBy = "Teacher", CreatedAt = now.AddMinutes(-10), StartedAt = now.AddMinutes(-10), ExpiresAt = now.AddMinutes(-1), Status = DeviceCommandStatuses.Dispatched }); // 3. Dispatched command with ExpiresAt in future, but >2 minutes since StartedAt without result db.DeviceCommands.Add(new DeviceCommand { Id = dispatchedTimedOutId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "Reconnect", CreatedBy = "Teacher", CreatedAt = now.AddMinutes(-5), StartedAt = now.AddMinutes(-3), ExpiresAt = now.AddMinutes(5), Status = DeviceCommandStatuses.Dispatched }); // 4. Dispatched command that is fresh (<30 seconds) db.DeviceCommands.Add(new DeviceCommand { Id = dispatchedFreshId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "RefreshConfig", CreatedBy = "Teacher", CreatedAt = now.AddSeconds(-20), StartedAt = now.AddSeconds(-20), ExpiresAt = now.AddMinutes(5), Status = DeviceCommandStatuses.Dispatched }); await db.SaveChangesAsync(); } // Connect Teacher Alpha to TeacherHub to observe broadcasts var teacherAlphaToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); var receivedBroadcasts = new ConcurrentBag(); var teacherAlphaConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/teacher", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["Authorization"] = $"Bearer {teacherAlphaToken}"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); teacherAlphaConn.On("OnDeviceCommandStatusChanged", res => receivedBroadcasts.Add(res)); await teacherAlphaConn.StartAsync(); try { // Execute ProcessTimeoutsAsync on DeviceCommandTimeoutWorker var worker = new DeviceCommandTimeoutWorker( factory.Services.GetRequiredService(), factory.Services.GetRequiredService>(), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); var transitioned = await worker.ProcessTimeoutsAsync(); if (transitioned != 3) throw new Exception($"Expected 3 commands transitioned, but got {transitioned}"); // Verify in DB using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var c1 = await db.DeviceCommands.FindAsync(pendingExpiredId); if (c1?.Status != DeviceCommandStatuses.Expired) throw new Exception($"Expected {pendingExpiredId} to be Expired, got {c1?.Status}"); if (c1.CompletedAt == null) throw new Exception($"Expected {pendingExpiredId} CompletedAt to be set"); var c2 = await db.DeviceCommands.FindAsync(dispatchedExpiredId); if (c2?.Status != DeviceCommandStatuses.TimedOut) throw new Exception($"Expected {dispatchedExpiredId} to be TimedOut, got {c2?.Status}"); var c3 = await db.DeviceCommands.FindAsync(dispatchedTimedOutId); if (c3?.Status != DeviceCommandStatuses.TimedOut) throw new Exception($"Expected {dispatchedTimedOutId} to be TimedOut, got {c3?.Status}"); var c4 = await db.DeviceCommands.FindAsync(dispatchedFreshId); if (c4?.Status != DeviceCommandStatuses.Dispatched) throw new Exception($"Expected {dispatchedFreshId} to remain Dispatched, got {c4?.Status}"); } // Verify SignalR broadcasts were received var spin = 0; while (receivedBroadcasts.Count < 3 && spin < 40) { await Task.Delay(50); spin++; } if (receivedBroadcasts.Count < 3) throw new Exception($"Expected at least 3 broadcasts on TeacherHub, got {receivedBroadcasts.Count}"); var b1 = receivedBroadcasts.FirstOrDefault(b => b.CommandId == pendingExpiredId); if (b1 == null || b1.Status != DeviceCommandStatuses.Expired || b1.ResultCode != -2) throw new Exception($"Broadcast for pending expired command mismatch: {b1?.Status}, {b1?.ResultCode}"); var b2 = receivedBroadcasts.FirstOrDefault(b => b.CommandId == dispatchedExpiredId); if (b2 == null || b2.Status != DeviceCommandStatuses.TimedOut || b2.ResultCode != -4) throw new Exception($"Broadcast for dispatched expired command mismatch: {b2?.Status}, {b2?.ResultCode}"); var b3 = receivedBroadcasts.FirstOrDefault(b => b.CommandId == dispatchedTimedOutId); if (b3 == null || b3.Status != DeviceCommandStatuses.TimedOut || b3.ResultCode != -4) throw new Exception($"Broadcast for dispatched 2min-timeout command mismatch: {b3?.Status}, {b3?.ResultCode}"); } finally { await teacherAlphaConn.StopAsync(); } } private static async Task Test_15_Durable_Command_Ledger_Survives_Restart_And_Retains_Deduplication() { var tempFile = Path.Combine(Path.GetTempPath(), $"cn_ledger_test_{Guid.NewGuid():N}.json"); try { var cmdId = $"cmd_durable_{Guid.NewGuid():N}"; var expectedResult = new DeviceCommandResultDto { CommandId = cmdId, DeviceId = "dev_alpha_1", Status = DeviceCommandStatuses.Succeeded, ResultCode = 0, ResultMessage = "Durable ledger execution verified", ExecutionDurationMs = 77, Timestamp = DateTimeOffset.UtcNow }; // 1. Initial service lifecycle: run command and complete it var dedup1 = new DeviceCommandDeduplicator(slidingWindow: TimeSpan.FromHours(1), maxCapacity: 100, ledgerFilePath: tempFile); if (!dedup1.TryBeginExecution(cmdId)) throw new Exception("Initial TryBeginExecution failed"); dedup1.CompleteExecution(cmdId, expectedResult); if (!File.Exists(tempFile)) throw new Exception("Ledger file was not created on disk after execution"); // 2. Simulate service crash / restart: create a new Deduplicator instance from the same ledger file var dedup2 = new DeviceCommandDeduplicator(slidingWindow: TimeSpan.FromHours(1), maxCapacity: 100, ledgerFilePath: tempFile); // Must reject duplicate execution if (dedup2.TryBeginExecution(cmdId)) throw new Exception("Post-restart TryBeginExecution permitted duplicate execution of finished command!"); // Must return cached result if (!dedup2.TryGetCachedResult(cmdId, out var cached) || cached == null) throw new Exception("Post-restart TryGetCachedResult failed to return cached execution result"); if (cached.CommandId != cmdId || cached.Status != DeviceCommandStatuses.Succeeded || cached.ExecutionDurationMs != 77) throw new Exception($"Post-restart cached result mismatch: Duration={cached.ExecutionDurationMs}, Status={cached.Status}"); // 3. Verify in-flight command (crash before complete) preserves deduplication on restart var crashCmdId = $"cmd_crash_{Guid.NewGuid():N}"; if (!dedup2.TryBeginExecution(crashCmdId)) throw new Exception("TryBeginExecution failed for crash test command"); var dedup3 = new DeviceCommandDeduplicator(slidingWindow: TimeSpan.FromHours(1), maxCapacity: 100, ledgerFilePath: tempFile); if (dedup3.TryBeginExecution(crashCmdId)) throw new Exception("Deduplicator allowed re-execution of in-flight command after service restart!"); await Task.CompletedTask; } finally { if (File.Exists(tempFile)) { try { File.Delete(tempFile); } catch { } } } } private static async Task Test_16_Command_DurationMs_Entity_Persistence_And_Audit_Query() { using var factory = new CustomWebApplicationFactory(); await SeedBaselineAsync(factory); var client = factory.CreateClient(); var cmdId = $"cmd_dur_{Guid.NewGuid():N}"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.DeviceCommands.Add(new DeviceCommand { Id = cmdId, TenantId = "t_alpha", DeviceId = "dev_alpha_1", CommandType = "SelfCheck", CreatedBy = "Teacher", CreatedAt = DateTimeOffset.UtcNow, StartedAt = DateTimeOffset.UtcNow, ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(5), Status = DeviceCommandStatuses.Dispatched }); await db.SaveChangesAsync(); } // Connect device and report result with DurationMs = 385 var hubConn = new HubConnectionBuilder() .WithUrl("http://localhost/hubs/device", options => { options.HttpMessageHandlerFactory = _ => factory.Server.CreateHandler(); options.Headers["X-Device-Id"] = "dev_alpha_1"; options.Headers["X-Device-Token"] = "tok_alpha_1_secret"; options.Transports = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .Build(); await hubConn.StartAsync(); try { var reportDto = new DeviceCommandResultDto { CommandId = cmdId, DeviceId = "dev_alpha_1", Status = DeviceCommandStatuses.Succeeded, ResultCode = 0, ResultMessage = "SelfCheck executed cleanly", ExecutionDurationMs = 385, Timestamp = DateTimeOffset.UtcNow }; await hubConn.InvokeAsync("ReportCommandResult", reportDto); await Task.Delay(400); // 1. Verify DurationMs persisted in EF Core entity using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var entity = await db.DeviceCommands.FirstOrDefaultAsync(c => c.Id == cmdId); if (entity == null) throw new Exception("Entity not found in DB"); if (entity.DurationMs != 385) throw new Exception($"Expected entity DurationMs == 385, got {entity.DurationMs}"); } // 2. Verify DurationMs returned in Teacher API query (/api/device-command/{cmdId}) var teacherToken = CustomWebApplicationFactory.CreateJwt("u_teacher_a", "张老师", "Teacher", "t_alpha"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", teacherToken); var cmdResp = await client.GetFromJsonAsync($"/api/device-command/{cmdId}"); if (cmdResp == null) throw new Exception("Failed to get command via API"); if (cmdResp.DurationMs != 385) throw new Exception($"Expected API DTO DurationMs == 385, got {cmdResp.DurationMs}"); // 3. Verify DurationMs returned in /api/device-command/history var historyResp = await client.GetFromJsonAsync>("/api/device-command/history"); var item = historyResp?.FirstOrDefault(c => c.CommandId == cmdId); if (item == null) throw new Exception("Command not found in history list"); if (item.DurationMs != 385) throw new Exception($"Expected history item DurationMs == 385, got {item.DurationMs}"); } finally { await hubConn.StopAsync(); } } private static async Task Test_17_Fail_Closed_Ledger_Write_Failure_Blocks_Execution() { // Point ledger to an invalid/unwritable location on Windows to trigger write failure var invalidLedgerPath = @"Z:\NonExistentDrive_CN_Test\invalid\ledger.json"; var dedup = new DeviceCommandDeduplicator(ledgerFilePath: invalidLedgerPath); var cmdId = $"cmd_fail_write_{Guid.NewGuid():N}"; int executorInvocationCount = 0; // Attempt to claim var claimResult = dedup.TryDurablyClaim(cmdId); if (claimResult != CommandClaimResult.StorageError) throw new Exception($"Expected StorageError on write failure, got {claimResult}"); // Fail-closed: TryBeginExecution must also return false if (dedup.TryBeginExecution(cmdId)) throw new Exception("TryBeginExecution returned true despite disk write failure!"); // In-memory record must be rolled back: Count should be 0 if (dedup.Count != 0) throw new Exception($"Deduplicator did not rollback in-memory claim, Count={dedup.Count}"); // Side effect executor must not run if (claimResult == CommandClaimResult.Claimed) { executorInvocationCount++; } if (executorInvocationCount != 0) throw new Exception($"Executor was invoked {executorInvocationCount} times despite disk write failure!"); await Task.CompletedTask; } private static async Task Test_18_Fail_Closed_Corrupted_Ledger_On_Restart_Rejects_Commands() { var tempFile = Path.Combine(Path.GetTempPath(), $"cn_corrupt_test_{Guid.NewGuid():N}.json"); try { // Write corrupted truncated JSON to simulate sudden power cut or disk corruption await File.WriteAllTextAsync(tempFile, "{\"cmd_partial\": {\"IsCompleted\": true, \"Timestamp\": \"2026-09-22T00:00:00Z\", \"Result\": {\"CommandId\": \"incomplete"); var dedup = new DeviceCommandDeduplicator(ledgerFilePath: tempFile); if (!dedup.IsCorrupted) throw new Exception("Deduplicator failed to detect corrupted ledger file on startup!"); var testCmdId = $"cmd_corrupt_test_{Guid.NewGuid():N}"; int sideEffectCount = 0; // Must fail-closed on claim var claimResult = dedup.TryDurablyClaim(testCmdId); if (claimResult != CommandClaimResult.StorageError) throw new Exception($"Expected StorageError on corrupted ledger, got {claimResult}"); if (dedup.TryBeginExecution(testCmdId)) throw new Exception("TryBeginExecution permitted execution on corrupted ledger!"); if (claimResult == CommandClaimResult.Claimed) { sideEffectCount++; } if (sideEffectCount != 0) throw new Exception("Side effect executed despite corrupted ledger file!"); // Test empty file corruption as well var emptyTempFile = Path.Combine(Path.GetTempPath(), $"cn_empty_test_{Guid.NewGuid():N}.json"); try { await File.WriteAllTextAsync(emptyTempFile, " "); var dedupEmpty = new DeviceCommandDeduplicator(ledgerFilePath: emptyTempFile); if (!dedupEmpty.IsCorrupted) throw new Exception("Deduplicator failed to detect empty/whitespace ledger file as corrupted!"); if (dedupEmpty.TryDurablyClaim(testCmdId) != CommandClaimResult.StorageError) throw new Exception("Empty ledger did not fail-closed with StorageError!"); } finally { if (File.Exists(emptyTempFile)) { try { File.Delete(emptyTempFile); } catch { } } } } finally { if (File.Exists(tempFile)) { try { File.Delete(tempFile); } catch { } } } } #endregion }