using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using ClassNotify.Server.Configuration; using ClassNotify.Server.Data; using ClassNotify.Server.Domain.Entities; using ClassNotify.Server.Services; using ClassNotify.Shared.Models; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; namespace ClassNotify.FleetTests; /// /// ClassNotify P2.5 RC1-4 Phase 3: Small Staged Rollout (人工控制的小规模灰度升级) 严格门禁测试套件 /// 覆盖 U21 ~ U30 全部验收指标与两大演示场景 (Scenario 1 正常版本灰度放行 & Scenario 2 坏版金丝雀自愈停机) /// public static class DeviceUpdatePhase3Tests { public static async Task RunAllTestsAsync(Func, Task> runTest) { Console.WriteLine("\n=========================================================="); Console.WriteLine(" ClassNotify P2.5 RC1-4 Phase 3: Small Staged Rollout"); Console.WriteLine("=========================================================="); await runTest("U21: Batch RBAC Security (Anon 401, Teacher 403, SchoolAdmin 403, SuperAdmin 202)", Test_U21_BatchRbacSecurity); await runTest("U22: Batch Scope & Boundary Enforcement (1~5 devices, single tenant, canary required)", Test_U22_BatchScopeEnforcement); await runTest("U23: Batch Idempotency & Repeat Protection (Same ClientRequestId returns existing batch)", Test_U23_BatchIdempotency); await runTest("U24: Canary Isolation & Dispatch Gate (Only Canary released, others Held with 0 commands)", Test_U24_CanaryIsolation); await runTest("U25: Canary Failure Automatic Batch Stop (Canary RolledBack/Failed stops batch immediately)", Test_U25_CanaryFailureStop); await runTest("U26: Canary Succeeded & Manual Continue (Only SuperAdmin can continue, releasing held devices)", Test_U26_ManualContinue); await runTest("U27: Stop Safety & In-Flight Protection (Stopped batch prevents new runs, in-flight safe)", Test_U27_StopSafety); await runTest("U28: Mixed Outcome Truthfulness & Dynamic Summary Invariant (RolledBack != Succeeded)", Test_U28_DynamicSummaryInvariant); await runTest("U29: Server Restart & State Durability (DB durability, seamless continue across restart)", Test_U29_RestartDurability); await runTest("U30: Full End-to-End Regression & Dual Scenarios (Scenario 1 Good Rollout + Scenario 2 Bad Canary Stop)", Test_U30_FullDualScenarios); } #region Helpers private static string HashDeviceToken(string token) { using var sha = SHA256.Create(); var bytes = sha.ComputeHash(Encoding.UTF8.GetBytes(token)); return Convert.ToBase64String(bytes); } private static async Task EnsureUserAsync(ClassNotifyDbContext db, string id, string username, string role, string tenantId) { var existing = await db.Users.FirstOrDefaultAsync(u => u.Id == id); if (existing == null) { db.Users.Add(new User { Id = id, Username = username, FullName = username, Role = role, TenantId = tenantId, PasswordHash = "hash", IsLocked = false, CreatedAt = DateTimeOffset.UtcNow }); await db.SaveChangesAsync(); } } private static async Task SeedTenantAndDevicesAsync( ClassNotifyDbContext db, string tenantId, string tenantName, List deviceIds, string serviceVersion = "2.5.0") { var tenant = await db.Tenants.FirstOrDefaultAsync(t => t.Id == tenantId); if (tenant == null) { tenant = new Tenant { Id = tenantId, Name = tenantName, IsActive = true, CreatedAt = DateTimeOffset.UtcNow }; db.Tenants.Add(tenant); } var classroom = await db.Classrooms.FirstOrDefaultAsync(c => c.TenantId == tenantId && c.Id == $"c_{tenantId}"); if (classroom == null) { classroom = new Classroom { Id = $"c_{tenantId}", TenantId = tenantId, DisplayName = $"Classroom {tenantId}" }; db.Classrooms.Add(classroom); } foreach (var devId in deviceIds) { var device = await db.Devices.FirstOrDefaultAsync(d => d.Id == devId); if (device == null) { device = new Device { Id = devId, TenantId = tenantId, ClassroomId = classroom.Id, DeviceTokenHash = HashDeviceToken($"token_{devId}"), Hostname = $"HOST-{devId}", ServiceVersion = serviceVersion, ClientVersion = "2.5.0", ProtocolVersion = 2, LastHeartbeatAt = DateTimeOffset.UtcNow, IsActive = true, CreatedAt = DateTimeOffset.UtcNow }; db.Devices.Add(device); } } await db.SaveChangesAsync(); } private static async Task SeedReleaseAsync( ClassNotifyDbContext db, string releaseId, string version, bool isEnabled = true) { var release = await db.UpdateReleases.FirstOrDefaultAsync(r => r.Id == releaseId); if (release == null) { release = new UpdateRelease { Id = releaseId, Version = version, Architecture = "win-x64", PackageRelativePath = $"{releaseId}.zip", PackageSha256 = "dummy_sha256_" + releaseId, PackageSizeBytes = 1024, ManifestSignature = "dummy_signature_" + releaseId, IsEnabled = isEnabled, CreatedAtUtc = DateTimeOffset.UtcNow }; db.UpdateReleases.Add(release); await db.SaveChangesAsync(); } return release; } private static async Task TriggerWorkerAsync(CustomWebApplicationFactory factory) { using var scope = factory.Services.CreateScope(); var worker = new DeviceUpdateDispatchWorker( scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService(), NullLogger.Instance); var method = typeof(DeviceUpdateDispatchWorker).GetMethod("ProcessPendingUpdatesAsync", BindingFlags.NonPublic | BindingFlags.Instance); var task = (Task)method!.Invoke(worker, new object[] { CancellationToken.None })!; await task; } private static async Task GetBatchDtoAsync(HttpClient client, string batchId, string token) { var req = new HttpRequestMessage(HttpMethod.Get, $"/api/admin/update-batches/{batchId}"); req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); var res = await client.SendAsync(req); if (res.StatusCode != HttpStatusCode.OK) throw new Exception($"GetBatchDtoAsync failed for {batchId}: StatusCode={res.StatusCode}"); var dto = await res.Content.ReadFromJsonAsync(); return dto ?? throw new Exception($"Null dto returned for {batchId}"); } private static async Task AdvanceRunToSuccessAsync(HttpClient client, string runId, string deviceId) { var steps = new[] { UpdateStatuses.Verified, UpdateStatuses.Installing, UpdateStatuses.HealthChecking, UpdateStatuses.Succeeded }; foreach (var st in steps) { var rep = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{runId}/status") { Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = st }) }; rep.Headers.Add("X-Device-Id", deviceId); rep.Headers.Add("X-Device-Token", $"token_{deviceId}"); var res = await client.SendAsync(rep); if (res.StatusCode != HttpStatusCode.OK) throw new Exception($"AdvanceRunToSuccessAsync failed at step {st}: {res.StatusCode}"); } } private static async Task AdvanceRunToRollbackAsync(HttpClient client, string runId, string deviceId, string errCode = UpdateErrorCodes.HealthCheckFailed) { var steps = new[] { (UpdateStatuses.Verified, (string?)null), (UpdateStatuses.Installing, null), (UpdateStatuses.HealthChecking, null), (UpdateStatuses.RollingBack, errCode), (UpdateStatuses.RolledBack, errCode) }; foreach (var (st, err) in steps) { var rep = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{runId}/status") { Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = st, ErrorCode = err, ErrorMessage = err != null ? "Simulated rollback" : null }) }; rep.Headers.Add("X-Device-Id", deviceId); rep.Headers.Add("X-Device-Token", $"token_{deviceId}"); var res = await client.SendAsync(rep); if (res.StatusCode != HttpStatusCode.OK) throw new Exception($"AdvanceRunToRollbackAsync failed at step {st}: {res.StatusCode}"); } } #endregion #region Gate Tests /// /// U21: Batch RBAC Security /// 创建与操作灰度批次接口严格受 RBAC 保护: /// 匿名请求 401 Unauthorized; /// Teacher 与 SchoolAdmin (即使属于同校) 严格 403 Forbidden; /// 仅限系统超级管理员 (SuperAdmin) 允许发起和控制批次 (202 Accepted / 200 OK) /// private static async Task Test_U21_BatchRbacSecurity() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var tenantId = "t_u21"; var dev1 = "dev_u21_1"; var dev2 = "dev_u21_2"; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await SeedTenantAndDevicesAsync(db, tenantId, "School U21", new List { dev1, dev2 }); await SeedReleaseAsync(db, "rel_u21", "2.6.0"); await EnsureUserAsync(db, "usr_u21_t", "teacher_u21", "Teacher", tenantId); await EnsureUserAsync(db, "usr_u21_sa", "schooladmin_u21", "SchoolAdmin", tenantId); await EnsureUserAsync(db, "usr_u21_super", "superadmin_u21", "SuperAdmin", "global"); } var postUrl = "/api/admin/update-batches"; var reqPayload = new CreateUpdateBatchRequest { TenantId = tenantId, ReleaseId = "rel_u21", CanaryDeviceId = dev1, DeviceIds = new List { dev1, dev2 } }; // 1. 匿名 -> 401 var anonRes = await client.PostAsJsonAsync(postUrl, reqPayload); if (anonRes.StatusCode != HttpStatusCode.Unauthorized) throw new Exception($"Expected 401 for anonymous create batch, got {anonRes.StatusCode}"); // 2. Teacher -> 403 var tToken = factory.GenerateToken("usr_u21_t", "teacher_u21", "Teacher", tenantId); var tReq = new HttpRequestMessage(HttpMethod.Post, postUrl) { Content = JsonContent.Create(reqPayload) }; tReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tToken); var tRes = await client.SendAsync(tReq); if (tRes.StatusCode != HttpStatusCode.Forbidden) throw new Exception($"Expected 403 for Teacher create batch, got {tRes.StatusCode}"); // 3. SchoolAdmin -> 403 var saToken = factory.GenerateToken("usr_u21_sa", "schooladmin_u21", "SchoolAdmin", tenantId); var saReq = new HttpRequestMessage(HttpMethod.Post, postUrl) { Content = JsonContent.Create(reqPayload) }; saReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", saToken); var saRes = await client.SendAsync(saReq); if (saRes.StatusCode != HttpStatusCode.Forbidden) throw new Exception($"Expected 403 for SchoolAdmin create batch, got {saRes.StatusCode}"); // 4. SuperAdmin -> 202 Accepted var superToken = factory.GenerateToken("usr_u21_super", "superadmin_u21", "SuperAdmin", "global"); var superReq = new HttpRequestMessage(HttpMethod.Post, postUrl) { Content = JsonContent.Create(reqPayload) }; superReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var superRes = await client.SendAsync(superReq); if (superRes.StatusCode != HttpStatusCode.Accepted) throw new Exception($"Expected 202 for SuperAdmin create batch, got {superRes.StatusCode}"); var batchDto = await superRes.Content.ReadFromJsonAsync(); if (batchDto == null || string.IsNullOrWhiteSpace(batchDto.BatchId)) throw new Exception("Expected valid UpdateBatchDto returned from batch creation."); // 5. Teacher 查询批次 -> 403 var tGetReq = new HttpRequestMessage(HttpMethod.Get, $"/api/admin/update-batches/{batchDto.BatchId}"); tGetReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tToken); var tGetRes = await client.SendAsync(tGetReq); if (tGetRes.StatusCode != HttpStatusCode.Forbidden) throw new Exception($"Expected 403 for Teacher get batch, got {tGetRes.StatusCode}"); // 6. SuperAdmin 查询版本下拉 -> 200 var relReq = new HttpRequestMessage(HttpMethod.Get, "/api/admin/update-releases"); relReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var relRes = await client.SendAsync(relReq); if (relRes.StatusCode != HttpStatusCode.OK) throw new Exception($"Expected 200 for SuperAdmin get releases, got {relRes.StatusCode}"); } /// /// U22: Batch Scope & Boundary Enforcement /// 灰度批次严格限定设备规模 1 至 5 台; /// 超过 5 台、空列表、重复设备、未指定 Canary、跨租户混合设备一律拒绝 (400 InvalidBatchScope) /// private static async Task Test_U22_BatchScopeEnforcement() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u22_super", "superadmin", "SuperAdmin", "global"); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u22_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, "t_u22_a", "School A", new List { "d_a1", "d_a2", "d_a3", "d_a4", "d_a5", "d_a6" }); await SeedTenantAndDevicesAsync(db, "t_u22_b", "School B", new List { "d_b1" }); await SeedReleaseAsync(db, "rel_u22", "2.6.0"); } async Task AssertBadRequest(CreateUpdateBatchRequest req, string expectedError) { var httpReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(req) }; httpReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var res = await client.SendAsync(httpReq); if (res.StatusCode != HttpStatusCode.BadRequest) throw new Exception($"Expected 400 BadRequest, got {res.StatusCode}"); var body = await res.Content.ReadAsStringAsync(); if (!body.Contains(expectedError)) throw new Exception($"Expected error code '{expectedError}', got '{body}'"); } // 1. 超过 5 台 (6台) -> 拒绝 await AssertBadRequest(new CreateUpdateBatchRequest { TenantId = "t_u22_a", ReleaseId = "rel_u22", CanaryDeviceId = "d_a1", DeviceIds = new List { "d_a1", "d_a2", "d_a3", "d_a4", "d_a5", "d_a6" } }, UpdateErrorCodes.InvalidBatchScope); // 2. 空列表 (0台) -> 拒绝 await AssertBadRequest(new CreateUpdateBatchRequest { TenantId = "t_u22_a", ReleaseId = "rel_u22", CanaryDeviceId = "d_a1", DeviceIds = new List() }, UpdateErrorCodes.InvalidBatchScope); // 3. 包含重复设备标识 -> 拒绝 await AssertBadRequest(new CreateUpdateBatchRequest { TenantId = "t_u22_a", ReleaseId = "rel_u22", CanaryDeviceId = "d_a1", DeviceIds = new List { "d_a1", "d_a1" } }, UpdateErrorCodes.InvalidBatchScope); // 4. Canary 不在设备列表中 -> 拒绝 await AssertBadRequest(new CreateUpdateBatchRequest { TenantId = "t_u22_a", ReleaseId = "rel_u22", CanaryDeviceId = "d_a99_nonexistent", DeviceIds = new List { "d_a1", "d_a2" } }, UpdateErrorCodes.InvalidBatchScope); // 5. 跨租户混合设备 (d_a1 属于 School A, d_b1 属于 School B) -> 拒绝 await AssertBadRequest(new CreateUpdateBatchRequest { TenantId = "t_u22_a", ReleaseId = "rel_u22", CanaryDeviceId = "d_a1", DeviceIds = new List { "d_a1", "d_b1" } }, UpdateErrorCodes.InvalidBatchScope); // 6. 合法 1 台设备 (Canary 本身) -> 接受 var req1 = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = "t_u22_a", ReleaseId = "rel_u22", CanaryDeviceId = "d_a1", DeviceIds = new List { "d_a1" } }) }; req1.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var res1 = await client.SendAsync(req1); if (res1.StatusCode != HttpStatusCode.Accepted) throw new Exception($"Expected 202 for 1 device batch, got {res1.StatusCode}"); } /// /// U23: Batch Idempotency & Repeat Protection /// 携带相同 ClientRequestId 的批次创建请求返回相同批次 DTO (202 Accepted); /// 数据库中严格仅存在 1 条批次记录与 1 条审计日志; /// 存在正在升级设备时禁止创建新批次 (409 Conflict) /// private static async Task Test_U23_BatchIdempotency() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u23_super", "superadmin", "SuperAdmin", "global"); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u23_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, "t_u23", "School U23", new List { "d_u23_1", "d_u23_2" }); await SeedReleaseAsync(db, "rel_u23", "2.6.0"); } var clientReqId = "req_u23_client_idem_001"; var payload = new CreateUpdateBatchRequest { TenantId = "t_u23", ReleaseId = "rel_u23", CanaryDeviceId = "d_u23_1", DeviceIds = new List { "d_u23_1", "d_u23_2" }, ClientRequestId = clientReqId }; // 1. 第一次请求 -> 202 var req1 = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(payload) }; req1.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var res1 = await client.SendAsync(req1); if (res1.StatusCode != HttpStatusCode.Accepted) throw new Exception($"Expected 202, got {res1.StatusCode}"); var dto1 = await res1.Content.ReadFromJsonAsync(); // 2. 第二次请求相同 ClientRequestId -> 幂等 202 并返回相同 BatchId var req2 = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(payload) }; req2.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var res2 = await client.SendAsync(req2); if (res2.StatusCode != HttpStatusCode.Accepted) throw new Exception($"Expected 202 for idempotent request, got {res2.StatusCode}"); var dto2 = await res2.Content.ReadFromJsonAsync(); if (dto1!.BatchId != dto2!.BatchId) throw new Exception($"Expected same BatchId {dto1.BatchId}, got {dto2.BatchId}"); // 3. 校验数据库中仅有 1 个批次和 1 条创建审计记录 using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var count = await db.DeviceUpdateBatches.CountAsync(b => b.ClientRequestId == clientReqId); if (count != 1) throw new Exception($"Expected exactly 1 batch in DB, found {count}"); var auditCount = await db.AdminAuditEvents.CountAsync(a => a.TargetId == dto1.BatchId && a.Action == "DEVICE_UPDATE_BATCH_CREATED"); if (auditCount != 1) throw new Exception($"Expected exactly 1 audit event for batch, found {auditCount}"); } // 4. 并发活跃冲突校验: 使用不同 ClientRequestId 但相同设备创建新批次 -> 409 Conflict var payloadConflict = new CreateUpdateBatchRequest { TenantId = "t_u23", ReleaseId = "rel_u23", CanaryDeviceId = "d_u23_1", DeviceIds = new List { "d_u23_1", "d_u23_2" }, ClientRequestId = "req_u23_diff_client" }; var reqConflict = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(payloadConflict) }; reqConflict.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var resConflict = await client.SendAsync(reqConflict); if (resConflict.StatusCode != HttpStatusCode.Conflict) throw new Exception($"Expected 409 Conflict when devices have active updates, got {resConflict.StatusCode}"); } /// /// U24: Canary Isolation & Dispatch Gate /// 灰度批次创建后,首发金丝雀设备 DispatchGate 为 Released,其余 1~4 台物理置为 Held; /// 后台调度 Worker 严格只拉取 Released 设备下发 StartDeviceUpdate 指令; /// Held 设备物理接收 0 条指令并保持 PendingDispatch /// private static async Task Test_U24_CanaryIsolation() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u24_super", "superadmin", "SuperAdmin", "global"); var tenantId = "t_u24"; var canaryId = "d_u24_c"; var otherIds = new List { "d_u24_2", "d_u24_3", "d_u24_4", "d_u24_5" }; var allIds = new List { canaryId }.Concat(otherIds).ToList(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u24_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, tenantId, "School U24", allIds); await SeedReleaseAsync(db, "rel_u24", "2.6.0"); } // 创建 5 台设备批次 var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = tenantId, ReleaseId = "rel_u24", CanaryDeviceId = canaryId, DeviceIds = allIds }) }; createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var createRes = await client.SendAsync(createReq); if (createRes.StatusCode != HttpStatusCode.Accepted) throw new Exception($"Expected 202, got {createRes.StatusCode}"); var dto = await createRes.Content.ReadFromJsonAsync(); // 1. 验证创建时的门禁状态 using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == dto!.BatchId).ToListAsync(); var canaryRun = runs.First(r => r.DeviceId == canaryId); if (!canaryRun.IsCanary || canaryRun.DispatchGate != UpdateDispatchGates.Released) throw new Exception($"Expected Canary run to have DispatchGate Released, got {canaryRun.DispatchGate}"); var heldRuns = runs.Where(r => r.DeviceId != canaryId).ToList(); if (heldRuns.Count != 4 || heldRuns.Any(r => r.DispatchGate != UpdateDispatchGates.Held)) throw new Exception("Expected all 4 non-canary runs to have DispatchGate Held."); } // 2. 触发调度 Worker (所有设备均在线) await TriggerWorkerAsync(factory); // 3. 验证物理隔离:仅 Canary 获得分发 (Downloading),其余 4 台物理接收 0 条指令 (保持 PendingDispatch 且 AttemptCount == 0) using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == dto!.BatchId).ToListAsync(); var canaryRun = runs.First(r => r.DeviceId == canaryId); if (canaryRun.Status != UpdateStatuses.Downloading || canaryRun.DispatchedAtUtc == null) throw new Exception($"Expected Canary run to be dispatched (Downloading), got {canaryRun.Status}"); var nonCanaries = runs.Where(r => r.DeviceId != canaryId).ToList(); foreach (var r in nonCanaries) { if (r.Status != UpdateStatuses.PendingDispatch || r.DispatchedAtUtc != null || r.DispatchAttemptCount != 0) throw new Exception($"Physical isolation violated! Device {r.DeviceId} received dispatch: Status={r.Status}, AttemptCount={r.DispatchAttemptCount}"); } } } /// /// U25: Canary Failure Automatic Batch Stop /// 当金丝雀验证设备在升级过程中发生失败 (进入 RolledBack / Failed / RollbackFailed / TimedOut 终态); /// 批次升级任务立即自动转入 Stopped 终态; /// 其余 1~4 台受控设备继续物理保持 Held,绝对不接收任何更新指令 /// private static async Task Test_U25_CanaryFailureStop() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u25_super", "superadmin", "SuperAdmin", "global"); var tenantId = "t_u25"; var canaryId = "d_u25_canary"; var otherIds = new List { "d_u25_2", "d_u25_3", "d_u25_4" }; var allIds = new List { canaryId }.Concat(otherIds).ToList(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u25_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, tenantId, "School U25", allIds); await SeedReleaseAsync(db, "rel_u25", "2.6.0"); } var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = tenantId, ReleaseId = "rel_u25", CanaryDeviceId = canaryId, DeviceIds = allIds }) }; createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var createRes = await client.SendAsync(createReq); var batchDto = await createRes.Content.ReadFromJsonAsync(); // 1. Worker 分发 Canary await TriggerWorkerAsync(factory); string canaryRunId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var cRun = await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchDto!.BatchId && r.IsCanary); canaryRunId = cRun.Id; } // 2. 金丝雀经历健康检查失败并执行自动回滚,上报 RolledBack await AdvanceRunToRollbackAsync(client, canaryRunId, canaryId); // 3. 验证批次状态自动变为 Stopped var currentBatch = await GetBatchDtoAsync(client, batchDto!.BatchId, superToken); if (currentBatch.Status != UpdateBatchStatuses.Stopped) throw new Exception($"Expected batch status Stopped, got {currentBatch.Status}"); if (currentBatch.StoppedAtUtc == null) throw new Exception("Expected StoppedAtUtc to be populated."); // 4. 再次触发 Worker,验证其余 3 台设备依然保持 Held,未被下发 await TriggerWorkerAsync(factory); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == batchDto.BatchId).ToListAsync(); var others = runs.Where(r => !r.IsCanary).ToList(); if (others.Any(r => r.Status != UpdateStatuses.PendingDispatch || r.DispatchGate != UpdateDispatchGates.Held)) throw new Exception("Held devices were improperly dispatched after canary failure!"); } // 5. 对 Stopped 批次调用 continue -> 400 BatchInvalidState var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/continue"); contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var contRes = await client.SendAsync(contReq); if (contRes.StatusCode != HttpStatusCode.BadRequest) throw new Exception($"Expected 400 for continue on stopped batch, got {contRes.StatusCode}"); } /// /// U26: Canary Succeeded & Manual Continue /// 金丝雀成功升级后,批次状态自动转入 AwaitingContinue; /// 严禁自动继续,必须等待 SuperAdmin 人工点击放行 (POST /api/admin/update-batches/{id}/continue); /// 放行后 Held 设备原子翻转为 Released,批次转为 RollingOut 并自动继续分发 /// private static async Task Test_U26_ManualContinue() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u26_super", "superadmin", "SuperAdmin", "global"); var tenantId = "t_u26"; var canaryId = "d_u26_c"; var otherIds = new List { "d_u26_2", "d_u26_3" }; var allIds = new List { canaryId }.Concat(otherIds).ToList(); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u26_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, tenantId, "School U26", allIds); await SeedReleaseAsync(db, "rel_u26", "2.6.0"); } var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = tenantId, ReleaseId = "rel_u26", CanaryDeviceId = canaryId, DeviceIds = allIds }) }; createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var createRes = await client.SendAsync(createReq); var batchDto = await createRes.Content.ReadFromJsonAsync(); // 1. Canary 执行升级并上报 Succeeded await TriggerWorkerAsync(factory); string canaryRunId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var cRun = await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchDto!.BatchId && r.IsCanary); canaryRunId = cRun.Id; } await AdvanceRunToSuccessAsync(client, canaryRunId, canaryId); // 2. 验证批次状态变为 AwaitingContinue var currentBatch = await GetBatchDtoAsync(client, batchDto!.BatchId, superToken); if (currentBatch.Status != UpdateBatchStatuses.AwaitingContinue) throw new Exception($"Expected batch status AwaitingContinue, got {currentBatch.Status}"); // 验证其余设备依然是 Held (零自动推进) if (currentBatch.Summary.Held != 2 || currentBatch.Summary.Succeeded != 1) throw new Exception($"Expected Held=2, Succeeded=1, got Held={currentBatch.Summary.Held}, Succeeded={currentBatch.Summary.Succeeded}"); // 3. SuperAdmin 人工点击放行 var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/continue"); contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var contRes = await client.SendAsync(contReq); if (contRes.StatusCode != HttpStatusCode.OK) throw new Exception($"Expected 200 OK for continue batch, got {contRes.StatusCode}"); var afterContBatch = await contRes.Content.ReadFromJsonAsync(); if (afterContBatch!.Status != UpdateBatchStatuses.RollingOut) throw new Exception($"Expected batch status RollingOut, got {afterContBatch.Status}"); // 4. 验证 Held 设备门禁全部转为 Released using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == batchDto.BatchId && !r.IsCanary).ToListAsync(); if (runs.Any(r => r.DispatchGate != UpdateDispatchGates.Released)) throw new Exception("Not all held devices were flipped to Released!"); // 验证审计日志 var audit = await db.AdminAuditEvents.FirstOrDefaultAsync(a => a.TargetId == batchDto.BatchId && a.Action == "DEVICE_UPDATE_BATCH_CONTINUED"); if (audit == null) throw new Exception("Continue audit event missing."); } // 5. 触发 Worker,验证 2 台新放行设备立即开始分发 (Downloading) await TriggerWorkerAsync(factory); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == batchDto.BatchId && !r.IsCanary).ToListAsync(); if (runs.Any(r => r.Status != UpdateStatuses.Downloading)) throw new Exception("Released runs were not dispatched by worker!"); } } /// /// U27: Stop Safety & In-Flight Protection /// 管理员人工停止批次 (POST /api/admin/update-batches/{id}/stop); /// Held 状态的任务物理保持阻断; /// 已经在执行中 (Downloading, Installing, HealthChecking) 的任务安全放行直至终态,绝不粗暴杀死 /// private static async Task Test_U27_StopSafety() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u27_super", "superadmin", "SuperAdmin", "global"); var tenantId = "t_u27"; var canaryId = "d_u27_c"; var devInFlight = "d_u27_flight"; var devHeld = "d_u27_held"; var allIds = new List { canaryId, devInFlight, devHeld }; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u27_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, tenantId, "School U27", allIds); await SeedReleaseAsync(db, "rel_u27", "2.6.0"); } // 1. 创建批次并完成 Canary 放行 var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = tenantId, ReleaseId = "rel_u27", CanaryDeviceId = canaryId, DeviceIds = allIds }) }; createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var createRes = await client.SendAsync(createReq); var batchDto = await createRes.Content.ReadFromJsonAsync(); // Canary 成功 await TriggerWorkerAsync(factory); string canaryRunId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); canaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchDto!.BatchId && r.IsCanary)).Id; } await AdvanceRunToSuccessAsync(client, canaryRunId, canaryId); // 放行 var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto!.BatchId}/continue"); contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); await client.SendAsync(contReq); // 调度两台放行设备,并将其中一台推进到 Installing await TriggerWorkerAsync(factory); string flightRunId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); flightRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.DeviceId == devInFlight)).Id; var heldRun = await db.DeviceUpdateRuns.FirstAsync(r => r.DeviceId == devHeld); // 模拟将 devHeld 恢复为 Held (以测试 Held 保护) heldRun.DispatchGate = UpdateDispatchGates.Held; heldRun.Status = UpdateStatuses.PendingDispatch; await db.SaveChangesAsync(); } var repVer = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status") { Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.Verified }) }; repVer.Headers.Add("X-Device-Id", devInFlight); repVer.Headers.Add("X-Device-Token", $"token_{devInFlight}"); await client.SendAsync(repVer); var repInst = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status") { Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.Installing }) }; repInst.Headers.Add("X-Device-Id", devInFlight); repInst.Headers.Add("X-Device-Token", $"token_{devInFlight}"); await client.SendAsync(repInst); // 2. 管理员调用 StopBatch var stopReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/stop"); stopReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var stopRes = await client.SendAsync(stopReq); if (stopRes.StatusCode != HttpStatusCode.OK) throw new Exception($"Expected 200 OK for stop batch, got {stopRes.StatusCode}"); var stoppedDto = await stopRes.Content.ReadFromJsonAsync(); if (stoppedDto!.Status != UpdateBatchStatuses.Stopped) throw new Exception($"Expected status Stopped, got {stoppedDto.Status}"); // 3. 验证安全门禁:执行中的设备允许继续上报并正常走向 Succeeded 终态 var repHc = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status") { Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.HealthChecking }) }; repHc.Headers.Add("X-Device-Id", devInFlight); repHc.Headers.Add("X-Device-Token", $"token_{devInFlight}"); await client.SendAsync(repHc); var reportFinish = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status") { Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.Succeeded }) }; reportFinish.Headers.Add("X-Device-Id", devInFlight); reportFinish.Headers.Add("X-Device-Token", $"token_{devInFlight}"); var reportFinishRes = await client.SendAsync(reportFinish); if (reportFinishRes.StatusCode != HttpStatusCode.OK) throw new Exception("In-flight update was improperly rejected after batch stop!"); // 4. 重复调用 StopBatch 是幂等的 var stopReq2 = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/stop"); stopReq2.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken); var stopRes2 = await client.SendAsync(stopReq2); if (stopRes2.StatusCode != HttpStatusCode.OK) throw new Exception("Second stop request was not handled idempotently."); } /// /// U28: Mixed Outcome Truthfulness & Dynamic Summary Invariant /// 灰度批次汇总数据严禁读取批次表持久化计数; /// 必须基于 DeviceUpdateRun 实时动态聚合; /// 严格守恒断言:Target == Held + Updating + Succeeded + RolledBack + Failed + RollbackFailed + TimedOut; /// RolledBack 严格独立统计,绝不混淆为 Succeeded /// private static async Task Test_U28_DynamicSummaryInvariant() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u28_super", "superadmin", "SuperAdmin", "global"); var tenantId = "t_u28"; var devCanary = "d_u28_c"; var devRb = "d_u28_rb"; var devFail = "d_u28_fail"; var devTimeout = "d_u28_to"; var devHeld = "d_u28_held"; var allIds = new List { devCanary, devRb, devFail, devTimeout, devHeld }; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u28_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, tenantId, "School U28", allIds); var rel = await SeedReleaseAsync(db, "rel_u28", "2.6.0"); var now = DateTimeOffset.UtcNow; var batch = new DeviceUpdateBatch { Id = "batch_u28_mixed", TenantId = tenantId, ReleaseId = rel.Id, CanaryDeviceId = devCanary, RequestedByUserId = "usr_u28_super", Status = UpdateBatchStatuses.RollingOut, TargetDeviceCount = 5, CreatedAtUtc = now }; var runs = new List { new() { Id = "r1", TenantId = tenantId, DeviceId = devCanary, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = true, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.Succeeded, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) }, new() { Id = "r2", TenantId = tenantId, DeviceId = devRb, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.RolledBack, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) }, new() { Id = "r3", TenantId = tenantId, DeviceId = devFail, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.Failed, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) }, new() { Id = "r4", TenantId = tenantId, DeviceId = devTimeout, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.TimedOut, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) }, new() { Id = "r5", TenantId = tenantId, DeviceId = devHeld, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Held, Status = UpdateStatuses.PendingDispatch, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) } }; db.DeviceUpdateBatches.Add(batch); db.DeviceUpdateRuns.AddRange(runs); await db.SaveChangesAsync(); } var dto = await GetBatchDtoAsync(client, "batch_u28_mixed", superToken); var s = dto.Summary; if (s.Target != 5) throw new Exception($"Expected Target=5, got {s.Target}"); if (s.Succeeded != 1) throw new Exception($"Expected Succeeded=1, got {s.Succeeded}"); if (s.RolledBack != 1) throw new Exception($"Expected RolledBack=1, got {s.RolledBack}"); if (s.Failed != 1) throw new Exception($"Expected Failed=1, got {s.Failed}"); if (s.TimedOut != 1) throw new Exception($"Expected TimedOut=1, got {s.TimedOut}"); if (s.Held != 1) throw new Exception($"Expected Held=1, got {s.Held}"); if (s.Updating != 0) throw new Exception($"Expected Updating=0, got {s.Updating}"); // 验证守恒 int sum = s.Held + s.Updating + s.Succeeded + s.RolledBack + s.Failed + s.RollbackFailed + s.TimedOut; if (s.Target != sum) throw new Exception($"Summary invariant violated! Target={s.Target}, Sum={sum}"); } /// /// U29: Server Restart & State Durability /// 模拟服务器意外崩溃重启: /// 批次状态、Canary 执行结果、未放行设备的 Held 门禁、审计日志等由数据库持久化保证; /// 重启后新服务实例无缝恢复 AwaitingContinue 状态并成功继续放行 /// private static async Task Test_U29_RestartDurability() { var sharedDbName = "classnotify_restart_test_" + Guid.NewGuid().ToString("N"); var tenantId = "t_u29"; var canaryId = "d_u29_c"; var dev2 = "d_u29_2"; string batchId; // 1. 实例 A 创建批次,Canary 完成并进入 AwaitingContinue using (var factoryA = new CustomWebApplicationFactory(sharedDbName)) { var clientA = factoryA.CreateClient(); var superTokenA = factoryA.GenerateToken("usr_u29_super", "superadmin", "SuperAdmin", "global"); using (var scope = factoryA.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u29_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, tenantId, "School U29", new List { canaryId, dev2 }); await SeedReleaseAsync(db, "rel_u29", "2.6.0"); } var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = tenantId, ReleaseId = "rel_u29", CanaryDeviceId = canaryId, DeviceIds = new List { canaryId, dev2 } }) }; createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superTokenA); var createRes = await clientA.SendAsync(createReq); var batchDto = await createRes.Content.ReadFromJsonAsync(); batchId = batchDto!.BatchId; await TriggerWorkerAsync(factoryA); string canaryRunId; using (var scope = factoryA.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); canaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchId && r.IsCanary)).Id; } await AdvanceRunToSuccessAsync(clientA, canaryRunId, canaryId); // 实例 A 停止运行 } // 2. 模拟服务器重启:启动全新实例 B 挂载相同数据库 using (var factoryB = new CustomWebApplicationFactory(sharedDbName)) { var clientB = factoryB.CreateClient(); var superTokenB = factoryB.GenerateToken("usr_u29_super", "superadmin", "SuperAdmin", "global"); // 查询批次状态 -> 完美保留 AwaitingContinue var batchDtoB = await GetBatchDtoAsync(clientB, batchId, superTokenB); if (batchDtoB.Status != UpdateBatchStatuses.AwaitingContinue) throw new Exception($"Expected durable status AwaitingContinue, got {batchDtoB.Status}"); if (batchDtoB.Summary.Held != 1 || batchDtoB.Summary.Succeeded != 1) throw new Exception($"Durable summary mismatch: Held={batchDtoB.Summary.Held}, Succeeded={batchDtoB.Summary.Succeeded}"); // 在重启后的实例 B 上继续执行人工放行 -> 成功 var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchId}/continue"); contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superTokenB); var contRes = await clientB.SendAsync(contReq); if (contRes.StatusCode != HttpStatusCode.OK) throw new Exception($"Expected 200 on continued batch after restart, got {contRes.StatusCode}"); var continuedDto = await contRes.Content.ReadFromJsonAsync(); if (continuedDto!.Status != UpdateBatchStatuses.RollingOut) throw new Exception($"Expected RollingOut status after restart, got {continuedDto.Status}"); } } /// /// U30: Full End-to-End Regression & Dual Scenarios /// 完整执行并验证两大核心业务场景: /// Scenario 1 (Good Release Rollout): 5 台设备,1 Canary 升级成功 -> AwaitingContinue -> 人工放行 -> 4 台全部成功 -> Completed /// Scenario 2 (Bad Canary Stop): 5 台设备,Canary 验证遇到故障自动回滚 (RolledBack) -> 批次立即自动 Stopped -> 4 台设备物理未触碰 (Held=4, Succeeded=0) /// private static async Task Test_U30_FullDualScenarios() { using var factory = new CustomWebApplicationFactory(); var client = factory.CreateClient(); var superToken = factory.GenerateToken("usr_u30_super", "superadmin", "SuperAdmin", "global"); // -------------------------------------------------------------------------------- // SCENARIO 1: 正常版本小规模灰度全流程演示 (Good Release Rollout) // -------------------------------------------------------------------------------- Console.WriteLine("\n [Scenario 1: Good Release Staged Rollout]"); var t1 = "t_u30_s1"; var c1 = "d_s1_canary"; var s1Devs = new List { c1, "d_s1_2", "d_s1_3", "d_s1_4", "d_s1_5" }; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u30_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, t1, "Good Rollout School", s1Devs, "2.5.0"); await SeedReleaseAsync(db, "rel_v260_good", "2.6.0"); } // 1. 创建 5 台设备批次 var s1CreateRes = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = t1, ReleaseId = "rel_v260_good", CanaryDeviceId = c1, DeviceIds = s1Devs }), Headers = { Authorization = new AuthenticationHeaderValue("Bearer", superToken) } }); if (s1CreateRes.StatusCode != HttpStatusCode.Accepted) throw new Exception("Scenario 1 batch creation failed."); var s1Batch = await s1CreateRes.Content.ReadFromJsonAsync(); // 2. 调度金丝雀设备 await TriggerWorkerAsync(factory); string s1CanaryRunId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); s1CanaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == s1Batch!.BatchId && r.IsCanary)).Id; } // 3. 金丝雀全流程成功 await AdvanceRunToSuccessAsync(client, s1CanaryRunId, c1); // 4. 验证批次进入 AwaitingContinue,其余 4 台处于 Held var s1Mid = await GetBatchDtoAsync(client, s1Batch!.BatchId, superToken); if (s1Mid.Status != UpdateBatchStatuses.AwaitingContinue || s1Mid.Summary.Held != 4 || s1Mid.Summary.Succeeded != 1) throw new Exception($"Scenario 1 Canary did not gate into AwaitingContinue: Status={s1Mid.Status}, Held={s1Mid.Summary.Held}, Succeeded={s1Mid.Summary.Succeeded}"); // 5. 人工确认放行 var s1Cont = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{s1Batch.BatchId}/continue") { Headers = { Authorization = new AuthenticationHeaderValue("Bearer", superToken) } }); if (s1Cont.StatusCode != HttpStatusCode.OK) throw new Exception("Scenario 1 continue failed."); // 6. 调度放行后的 4 台设备 await TriggerWorkerAsync(factory); List remainingRunIds; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); remainingRunIds = await db.DeviceUpdateRuns .Where(r => r.BatchId == s1Batch.BatchId && !r.IsCanary) .Select(r => r.Id) .ToListAsync(); } // 7. 4 台设备全部升级成功 foreach (var runId in remainingRunIds) { string devId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); devId = (await db.DeviceUpdateRuns.FirstAsync(r => r.Id == runId)).DeviceId; } await AdvanceRunToSuccessAsync(client, runId, devId); } // 8. 触发调度协调,批次最终转入 Completed await TriggerWorkerAsync(factory); var s1Final = await GetBatchDtoAsync(client, s1Batch.BatchId, superToken); if (s1Final.Status != UpdateBatchStatuses.Completed) throw new Exception($"Expected Scenario 1 batch Completed, got {s1Final.Status}"); if (s1Final.Summary.Target != 5 || s1Final.Summary.Succeeded != 5 || s1Final.Summary.Held != 0) throw new Exception($"Expected all 5 Succeeded, got Succeeded={s1Final.Summary.Succeeded}, Held={s1Final.Summary.Held}"); Console.WriteLine(" -> Scenario 1 PASSED: 5 devices upgraded to 2.6.0, Batch: Completed."); // -------------------------------------------------------------------------------- // SCENARIO 2: 坏版金丝雀自动回滚熔断停机全流程演示 (Bad Canary Automatic Rollback & Stop) // -------------------------------------------------------------------------------- Console.WriteLine("\n [Scenario 2: Bad Canary Automatic Rollback & Safe Batch Stop]"); var t2 = "t_u30_s2"; var c2 = "d_s2_canary"; var s2Devs = new List { c2, "d_s2_2", "d_s2_3", "d_s2_4", "d_s2_5" }; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await EnsureUserAsync(db, "usr_u30_super", "superadmin", "SuperAdmin", "global"); await SeedTenantAndDevicesAsync(db, t2, "Fault Rollback School", s2Devs, "2.5.0"); await SeedReleaseAsync(db, "rel_v270_bad", "2.7.0"); } // 1. 创建 5 台设备批次 var s2CreateRes = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(new CreateUpdateBatchRequest { TenantId = t2, ReleaseId = "rel_v270_bad", CanaryDeviceId = c2, DeviceIds = s2Devs }), Headers = { Authorization = new AuthenticationHeaderValue("Bearer", superToken) } }); if (s2CreateRes.StatusCode != HttpStatusCode.Accepted) throw new Exception("Scenario 2 batch creation failed."); var s2Batch = await s2CreateRes.Content.ReadFromJsonAsync(); // 2. 调度金丝雀设备 await TriggerWorkerAsync(factory); string s2CanaryRunId; using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); s2CanaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == s2Batch!.BatchId && r.IsCanary)).Id; } // 3. 金丝雀安装后健康检查失败,自愈回滚并上报 RolledBack await AdvanceRunToRollbackAsync(client, s2CanaryRunId, c2); // 4. 验证批次自动转为 Stopped,其余 4 台物理未触碰 (Held=4, Succeeded=0, 接收 0 条指令) var s2Final = await GetBatchDtoAsync(client, s2Batch!.BatchId, superToken); if (s2Final.Status != UpdateBatchStatuses.Stopped) throw new Exception($"Expected Scenario 2 batch Stopped, got {s2Final.Status}"); if (s2Final.Summary.Target != 5 || s2Final.Summary.Held != 4 || s2Final.Summary.RolledBack != 1 || s2Final.Summary.Succeeded != 0) throw new Exception($"Scenario 2 summary violation: Target={s2Final.Summary.Target}, Held={s2Final.Summary.Held}, RolledBack={s2Final.Summary.RolledBack}, Succeeded={s2Final.Summary.Succeeded}"); // 5. 再次触发 Worker,确认未放行设备保持 0 分发 await TriggerWorkerAsync(factory); using (var scope = factory.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var nonCanaries = await db.DeviceUpdateRuns.Where(r => r.BatchId == s2Batch.BatchId && !r.IsCanary).ToListAsync(); if (nonCanaries.Any(r => r.Status != UpdateStatuses.PendingDispatch || r.DispatchGate != UpdateDispatchGates.Held)) throw new Exception("Non-canary devices received update after canary rollback stop!"); } Console.WriteLine(" -> Scenario 2 PASSED: Canary safely rolled back to 2.5.0, Batch: Stopped, 4 devices Held with 0 updates."); } #endregion }