using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using ClassNotify.Server.Data; using ClassNotify.Server.Domain.Entities; using ClassNotify.Server.Services; using ClassNotify.Shared.Models; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; namespace ClassNotify.FleetTests; public static class SelfCheckBatchTests { public static async Task RunAllTestsAsync(Func, Task> runTest) { Console.WriteLine("\n=========================================================="); Console.WriteLine(" ClassNotify P2.5 RC1-2: SelfCheck Fleet Operations Suite"); Console.WriteLine("=========================================================="); await runTest("SCB1: Batch Creation (20 devices -> 1 batch, 20 unique runs, target=20)", Test_SCB1_BatchCreation); await runTest("SCB2: Idempotency (same ClientRequestId returns existing batch, 0 duplicate)", Test_SCB2_Idempotency); await runTest("SCB3: RBAC Access Control (Anon 401, Teacher 403, SchoolAdmin 403, SuperAdmin 200)", Test_SCB3_RBAC); await runTest("SCB4: Audit Transaction (Atomic audit insertion & rollback guarantee)", Test_SCB4_AuditTransaction); await runTest("SCB5: Tenant-Qualified Isolation (Runs preserve tenant boundary across schools)", Test_SCB5_TenantQualifiedIsolation); await runTest("SCB6: Bounded Concurrency (Bounded parallel dispatch queue)", Test_SCB6_BoundedConcurrency); await runTest("SCB7: Server Restart Recovery (PendingDispatch picked up after restart)", Test_SCB7_ServerRestartRecovery); await runTest("SCB8: Offline Truthfulness (Offline device -> Unreachable, null OverallStatus)", Test_SCB8_OfflineTruthfulness); await runTest("SCB9: Result Authorization (Device A posting Device B run -> 403 Forbidden)", Test_SCB9_ResultAuthorization); await runTest("SCB10: Result Idempotency (First-wins & duplicate idempotent rejection)", Test_SCB10_ResultIdempotency); await runTest("SCB11: Timeout Terminal State (Late result rejected with RunExpired)", Test_SCB11_TimeoutTerminalState); await runTest("SCB12: Aggregation Conservation (Pending + Healthy + Warn + Fail + Unreach + Timeout == Target)", Test_SCB12_AggregationConservation); await runTest("SCB13: Probe Drill-down (probe=ipc_pipe&health=Failed returns exact matching subset)", Test_SCB13_ProbeDrillDown); await runTest("SCB14: Partial Failure (Mixed health states accurately aggregated in summary & probes)", Test_SCB14_PartialFailure); await runTest("SCB15: No Control Side Effects (Observe-only self check, zero destructive commands)", Test_SCB15_NoControlSideEffects); } private static string HashToken(string token) { using var sha = System.Security.Cryptography.SHA256.Create(); var bytes = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(token)); return Convert.ToBase64String(bytes); } private static async Task Test_SCB1_BatchCreation() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var token = factory.GenerateToken("usr_super", "admin", "SuperAdmin", "global"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); // Seed 20 devices across 2 tenants using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scb1_a", Name = "学校A", IsActive = true }); db.Tenants.Add(new Tenant { Id = "t_scb1_b", Name = "学校B", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scb1_a", TenantId = "t_scb1_a", DisplayName = "班级A" }); db.Classrooms.Add(new Classroom { Id = "c_scb1_b", TenantId = "t_scb1_b", DisplayName = "班级B" }); for (int i = 1; i <= 10; i++) { db.Devices.Add(new Device { Id = $"dev_scb1_a_{i}", TenantId = "t_scb1_a", ClassroomId = "c_scb1_a", DeviceTokenHash = $"hash_a_{i}", IsActive = true }); db.Devices.Add(new Device { Id = $"dev_scb1_b_{i}", TenantId = "t_scb1_b", ClassroomId = "c_scb1_b", DeviceTokenHash = $"hash_b_{i}", IsActive = true }); } await db.SaveChangesAsync(); } var devIds = new List(); for (int i = 1; i <= 10; i++) { devIds.Add($"dev_scb1_a_{i}"); devIds.Add($"dev_scb1_b_{i}"); } var req = new CreateSelfCheckBatchRequest { ClientRequestId = "req_scb1_unique_001", DeviceIds = devIds }; var resp = await client.PostAsJsonAsync("/api/admin/self-check-batches", req); if (resp.StatusCode != HttpStatusCode.OK) { var err = await resp.Content.ReadAsStringAsync(); throw new Exception($"Failed to create batch: {resp.StatusCode} - {err}"); } var created = await resp.Content.ReadFromJsonAsync(); if (created == null || string.IsNullOrEmpty(created.BatchId) || created.TargetDeviceCount != 20) { throw new Exception($"Invalid batch created: count={created?.TargetDeviceCount}"); } using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var batch = await db.DeviceSelfCheckBatches.FirstOrDefaultAsync(b => b.Id == created.BatchId); if (batch == null || batch.TargetDeviceCount != 20) { throw new Exception("Batch record not found in DB or target count mismatch"); } var runs = await db.DeviceSelfChecks.Where(r => r.BatchId == created.BatchId).ToListAsync(); if (runs.Count != 20) { throw new Exception($"Expected 20 unique runs, found {runs.Count}"); } var distinctCorrs = runs.Select(r => r.CorrelationId).Distinct().Count(); if (distinctCorrs != 20) { throw new Exception($"CorrelationId collision: expected 20 unique correlation IDs, got {distinctCorrs}"); } } } private static async Task Test_SCB2_Idempotency() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var token = factory.GenerateToken("usr_super", "admin", "SuperAdmin", "global"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scb2", Name = "幂等测试校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scb2", TenantId = "t_scb2", DisplayName = "测试班" }); db.Devices.Add(new Device { Id = "dev_scb2_1", TenantId = "t_scb2", ClassroomId = "c_scb2", DeviceTokenHash = "h1", IsActive = true }); db.Devices.Add(new Device { Id = "dev_scb2_2", TenantId = "t_scb2", ClassroomId = "c_scb2", DeviceTokenHash = "h2", IsActive = true }); await db.SaveChangesAsync(); } var req = new CreateSelfCheckBatchRequest { ClientRequestId = "fixed_client_req_id_12345", DeviceIds = ["dev_scb2_1", "dev_scb2_2"] }; // First call var resp1 = await client.PostAsJsonAsync("/api/admin/self-check-batches", req); if (!resp1.IsSuccessStatusCode) throw new Exception("First call failed"); var b1 = await resp1.Content.ReadFromJsonAsync(); // Second call with same ClientRequestId var resp2 = await client.PostAsJsonAsync("/api/admin/self-check-batches", req); if (!resp2.IsSuccessStatusCode) throw new Exception("Second call failed"); var b2 = await resp2.Content.ReadFromJsonAsync(); if (b1?.BatchId != b2?.BatchId) { throw new Exception($"Idempotency violated: BatchId1={b1?.BatchId}, BatchId2={b2?.BatchId}"); } using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var batches = await db.DeviceSelfCheckBatches.Where(b => b.ClientRequestId == req.ClientRequestId).ToListAsync(); if (batches.Count != 1) { throw new Exception($"Expected 1 batch in DB, found {batches.Count}"); } } // --- Simultaneous Concurrent Creation Test --- var concReq = new CreateSelfCheckBatchRequest { ClientRequestId = "concurrent_client_req_" + Guid.NewGuid().ToString("N"), DeviceIds = ["dev_scb2_1", "dev_scb2_2"] }; var post1 = Task.Run(async () => { var c = factory.CreateClient(); c.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); return await c.PostAsJsonAsync("/api/admin/self-check-batches", concReq); }); var post2 = Task.Run(async () => { var c = factory.CreateClient(); c.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); return await c.PostAsJsonAsync("/api/admin/self-check-batches", concReq); }); var concResps = await Task.WhenAll(post1, post2); if (!concResps[0].IsSuccessStatusCode || !concResps[1].IsSuccessStatusCode) { throw new Exception($"Concurrent batch creation failed: resp0={concResps[0].StatusCode}, resp1={concResps[1].StatusCode}"); } var concB1 = await concResps[0].Content.ReadFromJsonAsync(); var concB2 = await concResps[1].Content.ReadFromJsonAsync(); if (concB1?.BatchId != concB2?.BatchId) { throw new Exception($"Concurrent idempotency violated: BatchId1={concB1?.BatchId}, BatchId2={concB2?.BatchId}"); } using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var count = await db.DeviceSelfCheckBatches.CountAsync(b => b.ClientRequestId == concReq.ClientRequestId); if (count != 1) { throw new Exception($"Expected exactly 1 batch in DB after concurrent creation, found {count}"); } } } private static async Task Test_SCB3_RBAC() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var anonReq = new HttpRequestMessage(HttpMethod.Get, "/api/admin/self-check-batches"); var anonResp = await client.SendAsync(anonReq); if (anonResp.StatusCode != HttpStatusCode.Unauthorized) { throw new Exception($"Anon expected 401, got {anonResp.StatusCode}"); } var teacherToken = factory.GenerateToken("usr_101", "teacher_zhang", "Teacher", "t_beijing_no4"); var tReq = new HttpRequestMessage(HttpMethod.Get, "/api/admin/self-check-batches"); tReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", teacherToken); var tResp = await client.SendAsync(tReq); if (tResp.StatusCode != HttpStatusCode.Forbidden) { throw new Exception($"Teacher expected 403, got {tResp.StatusCode}"); } var schoolAdminToken = factory.GenerateToken("usr_admin_bj", "admin_bj4", "SchoolAdmin", "t_beijing_no4"); var sReq = new HttpRequestMessage(HttpMethod.Get, "/api/admin/self-check-batches"); sReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", schoolAdminToken); var sResp = await client.SendAsync(sReq); if (sResp.StatusCode != HttpStatusCode.Forbidden) { throw new Exception($"SchoolAdmin expected 403, got {sResp.StatusCode}"); } var superToken = factory.GenerateToken("usr_super", "admin", "SuperAdmin", "global"); var superReq = new HttpRequestMessage(HttpMethod.Get, "/api/admin/self-check-batches"); superReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var superResp = await client.SendAsync(superReq); if (superResp.StatusCode != HttpStatusCode.OK) { throw new Exception($"SuperAdmin expected 200, got {superResp.StatusCode}"); } } private static async Task Test_SCB4_AuditTransaction() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var token = factory.GenerateToken("usr_super", "admin", "SuperAdmin", "global"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scb4", Name = "审计测试校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scb4", TenantId = "t_scb4", DisplayName = "测试班" }); db.Devices.Add(new Device { Id = "dev_scb4_1", TenantId = "t_scb4", ClassroomId = "c_scb4", DeviceTokenHash = "h1", IsActive = true }); await db.SaveChangesAsync(); } var req = new CreateSelfCheckBatchRequest { ClientRequestId = "audit_trans_001", DeviceIds = ["dev_scb4_1"] }; var resp = await client.PostAsJsonAsync("/api/admin/self-check-batches", req); if (!resp.IsSuccessStatusCode) throw new Exception("Batch creation failed"); var created = await resp.Content.ReadFromJsonAsync(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var audit = await db.AdminAuditEvents.FirstOrDefaultAsync(a => a.TargetId == created!.BatchId); if (audit == null || audit.Action != "SELF_CHECK_BATCH_REQUESTED") { throw new Exception("SELF_CHECK_BATCH_REQUESTED audit event missing from database"); } } } private static async Task Test_SCB5_TenantQualifiedIsolation() { using var factory = new CustomWebApplicationFactory(); using var scope = factory.Services.CreateScope(); var service = scope.ServiceProvider.GetRequiredService(); var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_iso_a", Name = "学校甲", IsActive = true }); db.Tenants.Add(new Tenant { Id = "t_iso_b", Name = "学校乙", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c1", TenantId = "t_iso_a", DisplayName = "班级1" }); db.Classrooms.Add(new Classroom { Id = "c1", TenantId = "t_iso_b", DisplayName = "班级1" }); db.Devices.Add(new Device { Id = "dev_iso_a", TenantId = "t_iso_a", ClassroomId = "c1", DeviceTokenHash = "h1", IsActive = true }); db.Devices.Add(new Device { Id = "dev_iso_b", TenantId = "t_iso_b", ClassroomId = "c1", DeviceTokenHash = "h2", IsActive = true }); await db.SaveChangesAsync(); var res = await service.CreateBatchAsync(new CreateSelfCheckBatchRequest { ClientRequestId = "req_iso_1", DeviceIds = ["dev_iso_a", "dev_iso_b"] }, "usr_admin", "127.0.0.1", CancellationToken.None); var runs = await db.DeviceSelfChecks.Where(r => r.BatchId == res.BatchId).ToListAsync(); var runA = runs.First(r => r.DeviceId == "dev_iso_a"); var runB = runs.First(r => r.DeviceId == "dev_iso_b"); if (runA.TenantId != "t_iso_a" || runB.TenantId != "t_iso_b") { throw new Exception($"Tenant isolation breach in runs: A={runA.TenantId}, B={runB.TenantId}"); } } private static async Task Test_SCB6_BoundedConcurrency() { // Tests that 100 devices can be registered in a batch, while max allowed is <= 100 using var factory = new CustomWebApplicationFactory(); using var scope = factory.Services.CreateScope(); var service = scope.ServiceProvider.GetRequiredService(); var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scale", Name = "规模测试校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scale", TenantId = "t_scale", DisplayName = "班级" }); var devIds = new List(); for (int i = 1; i <= 101; i++) { db.Devices.Add(new Device { Id = $"dev_scale_{i}", TenantId = "t_scale", ClassroomId = "c_scale", DeviceTokenHash = $"h_{i}", IsActive = true }); devIds.Add($"dev_scale_{i}"); } await db.SaveChangesAsync(); // 101 devices must be rejected (> 100 limit) try { await service.CreateBatchAsync(new CreateSelfCheckBatchRequest { ClientRequestId = "scale_req_101", DeviceIds = devIds }, "admin", "127.0.0.1", CancellationToken.None); throw new Exception("Expected ArgumentException for > 100 devices, but succeeded"); } catch (ArgumentException) { // Expected } // 100 devices must succeed var validBatch = await service.CreateBatchAsync(new CreateSelfCheckBatchRequest { ClientRequestId = "scale_req_100", DeviceIds = devIds.Take(100).ToList() }, "admin", "127.0.0.1", CancellationToken.None); if (validBatch.TargetDeviceCount != 100) { throw new Exception($"Expected target 100, got {validBatch.TargetDeviceCount}"); } } private static async Task Test_SCB7_ServerRestartRecovery() { using var factory = new CustomWebApplicationFactory(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_rec", Name = "恢复测试校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_rec", TenantId = "t_rec", DisplayName = "班级" }); db.Devices.Add(new Device { Id = "dev_rec_offline", TenantId = "t_rec", ClassroomId = "c_rec", DeviceTokenHash = "h1", IsActive = true }); var batch = new DeviceSelfCheckBatch { Id = "scb_crash_sim", CorrelationId = "corr_crash", ClientRequestId = "req_crash", CreatedByUserId = "admin", CreatedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddSeconds(60), TargetDeviceCount = 1, Status = "Running" }; db.DeviceSelfCheckBatches.Add(batch); var run = new DeviceSelfCheckRun { Id = "sc_crash_run", BatchId = batch.Id, TenantId = "t_rec", DeviceId = "dev_rec_offline", CorrelationId = "corr_run", RequestedByUserId = "admin", RequestedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = batch.DeadlineAtUtc, RunState = SelfCheckRunStates.PendingDispatch }; db.DeviceSelfChecks.Add(run); await db.SaveChangesAsync(); } // Wait a short moment for worker to pick up and process the pending run await Task.Delay(2500); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var run = await db.DeviceSelfChecks.FirstOrDefaultAsync(r => r.Id == "sc_crash_run"); if (run == null) throw new Exception("Run missing"); // Since dev_rec_offline is offline, worker should have transitioned it to Unreachable if (run.RunState != SelfCheckRunStates.Unreachable && run.RunState != SelfCheckRunStates.Dispatched) { throw new Exception($"Expected run to be processed after restart, but state is {run.RunState}"); } } } private static async Task Test_SCB8_OfflineTruthfulness() { using var factory = new CustomWebApplicationFactory(); using var scope = factory.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_off", Name = "离线真实性校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_off", TenantId = "t_off", DisplayName = "班级" }); db.Devices.Add(new Device { Id = "dev_off_true", TenantId = "t_off", ClassroomId = "c_off", DeviceTokenHash = "h1", IsActive = true, LastHeartbeatAt = DateTimeOffset.UtcNow.AddHours(-1) }); var batch = new DeviceSelfCheckBatch { Id = "scb_off", CorrelationId = "corr_off", ClientRequestId = "req_off", CreatedByUserId = "admin", CreatedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddSeconds(60), TargetDeviceCount = 1, Status = "Running" }; db.DeviceSelfCheckBatches.Add(batch); var run = new DeviceSelfCheckRun { Id = "sc_off_run", BatchId = batch.Id, TenantId = "t_off", DeviceId = "dev_off_true", CorrelationId = "corr_off_run", RequestedByUserId = "admin", RequestedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = batch.DeadlineAtUtc, RunState = SelfCheckRunStates.PendingDispatch }; db.DeviceSelfChecks.Add(run); await db.SaveChangesAsync(); // Allow worker to dispatch await Task.Delay(2500); using (var verifyScope = factory.Services.CreateScope()) { var verifyDb = verifyScope.ServiceProvider.GetRequiredService(); var updatedRun = await verifyDb.DeviceSelfChecks.AsNoTracking().FirstOrDefaultAsync(r => r.Id == "sc_off_run"); if (updatedRun?.RunState != SelfCheckRunStates.Unreachable) { throw new Exception($"Offline device must be Unreachable, got {updatedRun?.RunState}"); } if (updatedRun?.OverallStatus != null) { throw new Exception($"Offline device must not fake OverallStatus as {updatedRun?.OverallStatus}, must be null"); } } } private static async Task Test_SCB9_ResultAuthorization() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scb9", Name = "越权防范校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scb9", TenantId = "t_scb9", DisplayName = "班级" }); db.Devices.Add(new Device { Id = "dev_victim", TenantId = "t_scb9", ClassroomId = "c_scb9", DeviceTokenHash = HashToken("token_victim"), IsActive = true }); db.Devices.Add(new Device { Id = "dev_attacker", TenantId = "t_scb9", ClassroomId = "c_scb9", DeviceTokenHash = HashToken("token_attacker"), IsActive = true }); var run = new DeviceSelfCheckRun { Id = "sc_victim_run", TenantId = "t_scb9", DeviceId = "dev_victim", CorrelationId = "corr_v", RequestedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddMinutes(5), RunState = SelfCheckRunStates.Dispatched }; db.DeviceSelfChecks.Add(run); await db.SaveChangesAsync(); } // Attacker attempts to submit result for victim's run var payload = new DeviceSelfCheckResultDto { SelfCheckId = "sc_victim_run", DeviceId = "dev_victim", OverallStatus = SelfCheckStatus.Healthy }; var req = new HttpRequestMessage(HttpMethod.Post, "/api/device/self-check/sc_victim_run/result"); req.Headers.Add("X-Device-Id", "dev_attacker"); req.Headers.Add("X-Device-Token", "token_attacker"); req.Content = JsonContent.Create(payload); var resp = await client.SendAsync(req); if (resp.StatusCode != HttpStatusCode.Forbidden) { throw new Exception($"Cross-device result report expected 403 Forbidden, got {resp.StatusCode}"); } } private static async Task Test_SCB10_ResultIdempotency() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scb10", Name = "幂等首胜校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scb10", TenantId = "t_scb10", DisplayName = "班级" }); db.Devices.Add(new Device { Id = "dev_idem", TenantId = "t_scb10", ClassroomId = "c_scb10", DeviceTokenHash = HashToken("token_idem"), IsActive = true }); var run = new DeviceSelfCheckRun { Id = "sc_idem_run", TenantId = "t_scb10", DeviceId = "dev_idem", CorrelationId = "corr_idem", RequestedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddMinutes(5), RunState = SelfCheckRunStates.Dispatched }; db.DeviceSelfChecks.Add(run); await db.SaveChangesAsync(); } // First report: Healthy var p1 = new DeviceSelfCheckResultDto { SelfCheckId = "sc_idem_run", DeviceId = "dev_idem", OverallStatus = SelfCheckStatus.Healthy, Checks = [new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Healthy, Summary = "OK" }] }; var req1 = new HttpRequestMessage(HttpMethod.Post, "/api/device/self-check/sc_idem_run/result"); req1.Headers.Add("X-Device-Id", "dev_idem"); req1.Headers.Add("X-Device-Token", "token_idem"); req1.Content = JsonContent.Create(p1); var resp1 = await client.SendAsync(req1); if (!resp1.IsSuccessStatusCode) throw new Exception("First report failed"); // Second report: Try to overwrite with Failed var p2 = new DeviceSelfCheckResultDto { SelfCheckId = "sc_idem_run", DeviceId = "dev_idem", OverallStatus = SelfCheckStatus.Failed, Checks = [new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Failed, Summary = "Crash" }] }; var req2 = new HttpRequestMessage(HttpMethod.Post, "/api/device/self-check/sc_idem_run/result"); req2.Headers.Add("X-Device-Id", "dev_idem"); req2.Headers.Add("X-Device-Token", "token_idem"); req2.Content = JsonContent.Create(p2); var resp2 = await client.SendAsync(req2); if (!resp2.IsSuccessStatusCode) throw new Exception("Second report failed to return 200 idempotent OK"); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var run = await db.DeviceSelfChecks.FirstOrDefaultAsync(r => r.Id == "sc_idem_run"); if (run?.OverallStatus != "Healthy") { throw new Exception($"First-wins violated: first report Healthy was overwritten by {run?.OverallStatus}"); } } // --- Concurrent Race Condition Test --- // Seed another run for simultaneous race using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = "sc_race_run", TenantId = "t_scb10", DeviceId = "dev_idem", CorrelationId = "corr_race", RequestedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddMinutes(5), RunState = SelfCheckRunStates.Dispatched }); await db.SaveChangesAsync(); } var pA = new DeviceSelfCheckResultDto { SelfCheckId = "sc_race_run", DeviceId = "dev_idem", OverallStatus = SelfCheckStatus.Healthy, Checks = [new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Healthy, Summary = "OK" }] }; var pB = new DeviceSelfCheckResultDto { SelfCheckId = "sc_race_run", DeviceId = "dev_idem", OverallStatus = SelfCheckStatus.Failed, Checks = [new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Failed, Summary = "Failed crash" }] }; var taskA = Task.Run(async () => { var c = factory.CreateClient(); var req = new HttpRequestMessage(HttpMethod.Post, "/api/device/self-check/sc_race_run/result"); req.Headers.Add("X-Device-Id", "dev_idem"); req.Headers.Add("X-Device-Token", "token_idem"); req.Content = JsonContent.Create(pA); return await c.SendAsync(req); }); var taskB = Task.Run(async () => { var c = factory.CreateClient(); var req = new HttpRequestMessage(HttpMethod.Post, "/api/device/self-check/sc_race_run/result"); req.Headers.Add("X-Device-Id", "dev_idem"); req.Headers.Add("X-Device-Token", "token_idem"); req.Content = JsonContent.Create(pB); return await c.SendAsync(req); }); var responses = await Task.WhenAll(taskA, taskB); if (!responses[0].IsSuccessStatusCode || !responses[1].IsSuccessStatusCode) { throw new Exception($"Concurrent requests failed: resp0={responses[0].StatusCode}, resp1={responses[1].StatusCode}"); } using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var raceRun = await db.DeviceSelfChecks.AsNoTracking().FirstOrDefaultAsync(r => r.Id == "sc_race_run"); if (raceRun?.RunState != SelfCheckRunStates.Completed) { throw new Exception($"Race run expected Completed, got {raceRun?.RunState}"); } if (raceRun.OverallStatus != "Healthy" && raceRun.OverallStatus != "Failed") { throw new Exception($"Unexpected race run status: {raceRun.OverallStatus}"); } // Verify single valid JSON without corruption var parsed = JsonSerializer.Deserialize(raceRun.ResultJson!); if (parsed == null || parsed.OverallStatus.ToString() != raceRun.OverallStatus) { throw new Exception("ResultJson corrupted during concurrent first-wins race"); } } } private static async Task Test_SCB11_TimeoutTerminalState() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_scb11", Name = "超时终态校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_scb11", TenantId = "t_scb11", DisplayName = "班级" }); db.Devices.Add(new Device { Id = "dev_timeout", TenantId = "t_scb11", ClassroomId = "c_scb11", DeviceTokenHash = HashToken("token_timeout"), IsActive = true }); var run = new DeviceSelfCheckRun { Id = "sc_timeout_run", TenantId = "t_scb11", DeviceId = "dev_timeout", CorrelationId = "corr_t", RequestedAtUtc = DateTimeOffset.UtcNow.AddMinutes(-2), DeadlineAtUtc = DateTimeOffset.UtcNow.AddSeconds(-10), // expired! RunState = SelfCheckRunStates.TimedOut, Status = "Timeout" }; db.DeviceSelfChecks.Add(run); await db.SaveChangesAsync(); } // Late report arrives after deadline var p = new DeviceSelfCheckResultDto { SelfCheckId = "sc_timeout_run", DeviceId = "dev_timeout", OverallStatus = SelfCheckStatus.Healthy }; var req = new HttpRequestMessage(HttpMethod.Post, "/api/device/self-check/sc_timeout_run/result"); req.Headers.Add("X-Device-Id", "dev_timeout"); req.Headers.Add("X-Device-Token", "token_timeout"); req.Content = JsonContent.Create(p); var resp = await client.SendAsync(req); if (!resp.IsSuccessStatusCode) throw new Exception("Late report endpoint call failed"); var body = await resp.Content.ReadFromJsonAsync(); bool accepted = body.GetProperty("accepted").GetBoolean(); string reason = body.GetProperty("reason").GetString() ?? ""; if (accepted || reason != "RunExpired") { throw new Exception($"Expected accepted=false, reason=RunExpired, but got accepted={accepted}, reason={reason}"); } using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var run = await db.DeviceSelfChecks.FirstOrDefaultAsync(r => r.Id == "sc_timeout_run"); if (run?.RunState != SelfCheckRunStates.TimedOut) { throw new Exception($"TimedOut state was illegally modified to {run?.RunState}"); } } } private static async Task Test_SCB12_AggregationConservation() { using var factory = new CustomWebApplicationFactory(); using var scope = factory.Services.CreateScope(); var service = scope.ServiceProvider.GetRequiredService(); var db = scope.ServiceProvider.GetRequiredService(); var batch = new DeviceSelfCheckBatch { Id = "scb_conserv", CorrelationId = "corr_c", ClientRequestId = "req_c", CreatedByUserId = "admin", CreatedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddMinutes(5), TargetDeviceCount = 20, Status = "Running" }; db.DeviceSelfCheckBatches.Add(batch); // 5 Healthy, 3 Warning, 2 Failed, 4 Unreachable, 3 TimedOut, 3 Pending = 20 total for (int i = 1; i <= 5; i++) db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_h_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"d_h_{i}", RunState = SelfCheckRunStates.Completed, OverallStatus = "Healthy" }); for (int i = 1; i <= 3; i++) db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_w_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"d_w_{i}", RunState = SelfCheckRunStates.Completed, OverallStatus = "Warning" }); for (int i = 1; i <= 2; i++) db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_f_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"d_f_{i}", RunState = SelfCheckRunStates.Completed, OverallStatus = "Failed" }); for (int i = 1; i <= 4; i++) db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_u_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"d_u_{i}", RunState = SelfCheckRunStates.Unreachable }); for (int i = 1; i <= 3; i++) db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_to_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"d_to_{i}", RunState = SelfCheckRunStates.TimedOut }); for (int i = 1; i <= 3; i++) db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_p_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"d_p_{i}", RunState = SelfCheckRunStates.PendingDispatch }); await db.SaveChangesAsync(); var detail = await service.GetBatchAsync(batch.Id, CancellationToken.None); if (detail == null) throw new Exception("Batch detail is null"); var s = detail.Summary; int sum = s.Pending + s.Healthy + s.Warning + s.Failed + s.Unreachable + s.TimedOut; if (sum != s.Target || s.Target != 20) { throw new Exception($"Conservation invariant violated: sum={sum}, target={s.Target}"); } if (s.Completed != (s.Healthy + s.Warning + s.Failed) || s.Completed != 10) { throw new Exception($"Completed count mismatch: completed={s.Completed}, expected 10"); } } private static async Task Test_SCB13_ProbeDrillDown() { using var factory = new CustomWebApplicationFactory(); using var scope = factory.Services.CreateScope(); var service = scope.ServiceProvider.GetRequiredService(); var db = scope.ServiceProvider.GetRequiredService(); var batch = new DeviceSelfCheckBatch { Id = "scb_drill", CorrelationId = "corr_d", ClientRequestId = "req_d", CreatedByUserId = "admin", CreatedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddMinutes(5), TargetDeviceCount = 5, Status = "Completed" }; db.DeviceSelfCheckBatches.Add(batch); db.Tenants.Add(new Tenant { Id = "t", Name = "下钻测试校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_drill", TenantId = "t", DisplayName = "下钻班" }); for (int i = 1; i <= 3; i++) { db.Devices.Add(new Device { Id = $"dev_ipc_failed_{i}", TenantId = "t", ClassroomId = "c_drill", DeviceTokenHash = $"tok_{i}", IsActive = true }); } for (int i = 4; i <= 5; i++) { db.Devices.Add(new Device { Id = $"dev_tts_failed_{i}", TenantId = "t", ClassroomId = "c_drill", DeviceTokenHash = $"tok_{i}", IsActive = true }); } // Inject 3 devices with IPC Failed for (int i = 1; i <= 3; i++) { var res = new DeviceSelfCheckResultDto { SelfCheckId = $"r_drill_{i}", DeviceId = $"dev_ipc_failed_{i}", OverallStatus = SelfCheckStatus.Failed, Checks = [ new SelfCheckItemDto { Name = "ipc_pipe", Status = SelfCheckStatus.Failed, Summary = "Named pipe error" }, new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Healthy, Summary = "OK" } ] }; db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_drill_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"dev_ipc_failed_{i}", RunState = SelfCheckRunStates.Completed, OverallStatus = "Failed", ResultJson = JsonSerializer.Serialize(res) }); } // Inject 2 devices with TTS Failed (IPC healthy) for (int i = 4; i <= 5; i++) { var res = new DeviceSelfCheckResultDto { SelfCheckId = $"r_drill_{i}", DeviceId = $"dev_tts_failed_{i}", OverallStatus = SelfCheckStatus.Failed, Checks = [ new SelfCheckItemDto { Name = "ipc_pipe", Status = SelfCheckStatus.Healthy, Summary = "OK" }, new SelfCheckItemDto { Name = "tts_engine", Status = SelfCheckStatus.Failed, Summary = "Voice synthesizer crash" } ] }; db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = $"r_drill_{i}", BatchId = batch.Id, TenantId = "t", DeviceId = $"dev_tts_failed_{i}", RunState = SelfCheckRunStates.Completed, OverallStatus = "Failed", ResultJson = JsonSerializer.Serialize(res) }); } await db.SaveChangesAsync(); var queryResult = await service.GetBatchDevicesAsync(batch.Id, new SelfCheckBatchDeviceQuery { Probe = "ipc_pipe", Health = "Failed" }, CancellationToken.None); if (queryResult.Total != 3) { throw new Exception($"Expected 3 devices with IPC Failed, got {queryResult.Total}"); } var returnedIds = queryResult.Items.Select(x => x.DeviceId).OrderBy(x => x).ToList(); var expectedIds = new List { "dev_ipc_failed_1", "dev_ipc_failed_2", "dev_ipc_failed_3" }; if (!returnedIds.SequenceEqual(expectedIds)) { throw new Exception($"Drill-down returned unexpected devices: {string.Join(", ", returnedIds)}"); } } private static async Task Test_SCB14_PartialFailure() { using var factory = new CustomWebApplicationFactory(); using var scope = factory.Services.CreateScope(); var service = scope.ServiceProvider.GetRequiredService(); var db = scope.ServiceProvider.GetRequiredService(); var batch = new DeviceSelfCheckBatch { Id = "scb_partial", CorrelationId = "corr_p", ClientRequestId = "req_p", CreatedByUserId = "admin", CreatedAtUtc = DateTimeOffset.UtcNow, DeadlineAtUtc = DateTimeOffset.UtcNow.AddMinutes(5), TargetDeviceCount = 2, Status = "Completed" }; db.DeviceSelfCheckBatches.Add(batch); var r1 = new DeviceSelfCheckResultDto { SelfCheckId = "r_part_1", DeviceId = "d1", OverallStatus = SelfCheckStatus.Healthy, Checks = [ new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Healthy, Summary = "OK" }, new SelfCheckItemDto { Name = "clock_offset", Status = SelfCheckStatus.Healthy, Summary = "OK" } ] }; var r2 = new DeviceSelfCheckResultDto { SelfCheckId = "r_part_2", DeviceId = "d2", OverallStatus = SelfCheckStatus.Warning, Checks = [ new SelfCheckItemDto { Name = "service", Status = SelfCheckStatus.Healthy, Summary = "OK" }, new SelfCheckItemDto { Name = "clock_offset", Status = SelfCheckStatus.Warning, Summary = "Skew 4s" } ] }; db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = "r_part_1", BatchId = batch.Id, TenantId = "t", DeviceId = "d1", RunState = SelfCheckRunStates.Completed, OverallStatus = "Healthy", ResultJson = JsonSerializer.Serialize(r1) }); db.DeviceSelfChecks.Add(new DeviceSelfCheckRun { Id = "r_part_2", BatchId = batch.Id, TenantId = "t", DeviceId = "d2", RunState = SelfCheckRunStates.Completed, OverallStatus = "Warning", ResultJson = JsonSerializer.Serialize(r2) }); await db.SaveChangesAsync(); var detail = await service.GetBatchAsync(batch.Id, CancellationToken.None); if (detail == null) throw new Exception("Batch detail is null"); var clockProbe = detail.ProbeFailures.FirstOrDefault(p => p.Probe == "clock_offset"); if (clockProbe == null || clockProbe.WarningCount != 1 || clockProbe.FailedCount != 0) { throw new Exception($"Clock probe failure counts mismatch: Warn={clockProbe?.WarningCount}, Failed={clockProbe?.FailedCount}"); } } private static async Task Test_SCB15_NoControlSideEffects() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var token = factory.GenerateToken("usr_super", "admin", "SuperAdmin", "global"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); db.Tenants.Add(new Tenant { Id = "t_side", Name = "无副作用测试校", IsActive = true }); db.Classrooms.Add(new Classroom { Id = "c_side", TenantId = "t_side", DisplayName = "班级" }); db.Devices.Add(new Device { Id = "dev_side", TenantId = "t_side", ClassroomId = "c_side", DeviceTokenHash = "h_side", IsActive = true }); await db.SaveChangesAsync(); } var req = new CreateSelfCheckBatchRequest { ClientRequestId = "side_effect_check", DeviceIds = ["dev_side"] }; await client.PostAsJsonAsync("/api/admin/self-check-batches", req); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); // Verify no destructive commands were queued var destructiveCommands = await db.DeviceCommands .Where(c => c.CommandType == DeviceCommandTypes.RestartClient) .ToListAsync(); if (destructiveCommands.Count > 0) { throw new Exception($"Destructive command generated during SelfCheck: {destructiveCommands[0].CommandType}"); } var dev = await db.Devices.FirstOrDefaultAsync(d => d.Id == "dev_side"); if (!dev!.IsActive || dev.ClassroomId != "c_side") { throw new Exception("Device configuration modified during SelfCheck"); } } } }