using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using ClassNotify.Server.Configuration;
using ClassNotify.Server.Data;
using ClassNotify.Server.Domain.Entities;
using ClassNotify.Server.Services;
using ClassNotify.Shared.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace ClassNotify.FleetTests;
///
/// ClassNotify P2.5 RC1-4 Phase 3: Small Staged Rollout (人工控制的小规模灰度升级) 严格门禁测试套件
/// 覆盖 U21 ~ U30 全部验收指标与两大演示场景 (Scenario 1 正常版本灰度放行 & Scenario 2 坏版金丝雀自愈停机)
///
public static class DeviceUpdatePhase3Tests
{
public static async Task RunAllTestsAsync(Func, Task> runTest)
{
Console.WriteLine("\n==========================================================");
Console.WriteLine(" ClassNotify P2.5 RC1-4 Phase 3: Small Staged Rollout");
Console.WriteLine("==========================================================");
await runTest("U21: Batch RBAC Security (Anon 401, Teacher 403, SchoolAdmin 403, SuperAdmin 202)", Test_U21_BatchRbacSecurity);
await runTest("U22: Batch Scope & Boundary Enforcement (1~5 devices, single tenant, canary required)", Test_U22_BatchScopeEnforcement);
await runTest("U23: Batch Idempotency & Repeat Protection (Same ClientRequestId returns existing batch)", Test_U23_BatchIdempotency);
await runTest("U24: Canary Isolation & Dispatch Gate (Only Canary released, others Held with 0 commands)", Test_U24_CanaryIsolation);
await runTest("U25: Canary Failure Automatic Batch Stop (Canary RolledBack/Failed stops batch immediately)", Test_U25_CanaryFailureStop);
await runTest("U26: Canary Succeeded & Manual Continue (Only SuperAdmin can continue, releasing held devices)", Test_U26_ManualContinue);
await runTest("U27: Stop Safety & In-Flight Protection (Stopped batch prevents new runs, in-flight safe)", Test_U27_StopSafety);
await runTest("U28: Mixed Outcome Truthfulness & Dynamic Summary Invariant (RolledBack != Succeeded)", Test_U28_DynamicSummaryInvariant);
await runTest("U29: Server Restart & State Durability (DB durability, seamless continue across restart)", Test_U29_RestartDurability);
await runTest("U30: Full End-to-End Regression & Dual Scenarios (Scenario 1 Good Rollout + Scenario 2 Bad Canary Stop)", Test_U30_FullDualScenarios);
}
#region Helpers
private static string HashDeviceToken(string token)
{
using var sha = SHA256.Create();
var bytes = sha.ComputeHash(Encoding.UTF8.GetBytes(token));
return Convert.ToBase64String(bytes);
}
private static async Task EnsureUserAsync(ClassNotifyDbContext db, string id, string username, string role, string tenantId)
{
var existing = await db.Users.FirstOrDefaultAsync(u => u.Id == id);
if (existing == null)
{
db.Users.Add(new User
{
Id = id,
Username = username,
FullName = username,
Role = role,
TenantId = tenantId,
PasswordHash = "hash",
IsLocked = false,
CreatedAt = DateTimeOffset.UtcNow
});
await db.SaveChangesAsync();
}
}
private static async Task SeedTenantAndDevicesAsync(
ClassNotifyDbContext db,
string tenantId,
string tenantName,
List deviceIds,
string serviceVersion = "2.5.0")
{
var tenant = await db.Tenants.FirstOrDefaultAsync(t => t.Id == tenantId);
if (tenant == null)
{
tenant = new Tenant
{
Id = tenantId,
Name = tenantName,
IsActive = true,
CreatedAt = DateTimeOffset.UtcNow
};
db.Tenants.Add(tenant);
}
var classroom = await db.Classrooms.FirstOrDefaultAsync(c => c.TenantId == tenantId && c.Id == $"c_{tenantId}");
if (classroom == null)
{
classroom = new Classroom
{
Id = $"c_{tenantId}",
TenantId = tenantId,
DisplayName = $"Classroom {tenantId}"
};
db.Classrooms.Add(classroom);
}
foreach (var devId in deviceIds)
{
var device = await db.Devices.FirstOrDefaultAsync(d => d.Id == devId);
if (device == null)
{
device = new Device
{
Id = devId,
TenantId = tenantId,
ClassroomId = classroom.Id,
DeviceTokenHash = HashDeviceToken($"token_{devId}"),
Hostname = $"HOST-{devId}",
ServiceVersion = serviceVersion,
ClientVersion = "2.5.0",
ProtocolVersion = 2,
LastHeartbeatAt = DateTimeOffset.UtcNow,
IsActive = true,
CreatedAt = DateTimeOffset.UtcNow
};
db.Devices.Add(device);
}
}
await db.SaveChangesAsync();
}
private static async Task SeedReleaseAsync(
ClassNotifyDbContext db,
string releaseId,
string version,
bool isEnabled = true)
{
var release = await db.UpdateReleases.FirstOrDefaultAsync(r => r.Id == releaseId);
if (release == null)
{
release = new UpdateRelease
{
Id = releaseId,
Version = version,
Architecture = "win-x64",
PackageRelativePath = $"{releaseId}.zip",
PackageSha256 = "dummy_sha256_" + releaseId,
PackageSizeBytes = 1024,
ManifestSignature = "dummy_signature_" + releaseId,
IsEnabled = isEnabled,
CreatedAtUtc = DateTimeOffset.UtcNow
};
db.UpdateReleases.Add(release);
await db.SaveChangesAsync();
}
return release;
}
private static async Task TriggerWorkerAsync(CustomWebApplicationFactory factory)
{
using var scope = factory.Services.CreateScope();
var worker = new DeviceUpdateDispatchWorker(
scope.ServiceProvider.GetRequiredService(),
scope.ServiceProvider.GetRequiredService(),
NullLogger.Instance);
var method = typeof(DeviceUpdateDispatchWorker).GetMethod("ProcessPendingUpdatesAsync", BindingFlags.NonPublic | BindingFlags.Instance);
var task = (Task)method!.Invoke(worker, new object[] { CancellationToken.None })!;
await task;
}
private static async Task GetBatchDtoAsync(HttpClient client, string batchId, string token)
{
var req = new HttpRequestMessage(HttpMethod.Get, $"/api/admin/update-batches/{batchId}");
req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
var res = await client.SendAsync(req);
if (res.StatusCode != HttpStatusCode.OK)
throw new Exception($"GetBatchDtoAsync failed for {batchId}: StatusCode={res.StatusCode}");
var dto = await res.Content.ReadFromJsonAsync();
return dto ?? throw new Exception($"Null dto returned for {batchId}");
}
private static async Task AdvanceRunToSuccessAsync(HttpClient client, string runId, string deviceId)
{
var steps = new[] { UpdateStatuses.Verified, UpdateStatuses.Installing, UpdateStatuses.HealthChecking, UpdateStatuses.Succeeded };
foreach (var st in steps)
{
var rep = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{runId}/status")
{
Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = st })
};
rep.Headers.Add("X-Device-Id", deviceId);
rep.Headers.Add("X-Device-Token", $"token_{deviceId}");
var res = await client.SendAsync(rep);
if (res.StatusCode != HttpStatusCode.OK)
throw new Exception($"AdvanceRunToSuccessAsync failed at step {st}: {res.StatusCode}");
}
}
private static async Task AdvanceRunToRollbackAsync(HttpClient client, string runId, string deviceId, string errCode = UpdateErrorCodes.HealthCheckFailed)
{
var steps = new[]
{
(UpdateStatuses.Verified, (string?)null),
(UpdateStatuses.Installing, null),
(UpdateStatuses.HealthChecking, null),
(UpdateStatuses.RollingBack, errCode),
(UpdateStatuses.RolledBack, errCode)
};
foreach (var (st, err) in steps)
{
var rep = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{runId}/status")
{
Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = st, ErrorCode = err, ErrorMessage = err != null ? "Simulated rollback" : null })
};
rep.Headers.Add("X-Device-Id", deviceId);
rep.Headers.Add("X-Device-Token", $"token_{deviceId}");
var res = await client.SendAsync(rep);
if (res.StatusCode != HttpStatusCode.OK)
throw new Exception($"AdvanceRunToRollbackAsync failed at step {st}: {res.StatusCode}");
}
}
#endregion
#region Gate Tests
///
/// U21: Batch RBAC Security
/// 创建与操作灰度批次接口严格受 RBAC 保护:
/// 匿名请求 401 Unauthorized;
/// Teacher 与 SchoolAdmin (即使属于同校) 严格 403 Forbidden;
/// 仅限系统超级管理员 (SuperAdmin) 允许发起和控制批次 (202 Accepted / 200 OK)
///
private static async Task Test_U21_BatchRbacSecurity()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var tenantId = "t_u21";
var dev1 = "dev_u21_1";
var dev2 = "dev_u21_2";
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await SeedTenantAndDevicesAsync(db, tenantId, "School U21", new List { dev1, dev2 });
await SeedReleaseAsync(db, "rel_u21", "2.6.0");
await EnsureUserAsync(db, "usr_u21_t", "teacher_u21", "Teacher", tenantId);
await EnsureUserAsync(db, "usr_u21_sa", "schooladmin_u21", "SchoolAdmin", tenantId);
await EnsureUserAsync(db, "usr_u21_super", "superadmin_u21", "SuperAdmin", "global");
}
var postUrl = "/api/admin/update-batches";
var reqPayload = new CreateUpdateBatchRequest
{
TenantId = tenantId,
ReleaseId = "rel_u21",
CanaryDeviceId = dev1,
DeviceIds = new List { dev1, dev2 }
};
// 1. 匿名 -> 401
var anonRes = await client.PostAsJsonAsync(postUrl, reqPayload);
if (anonRes.StatusCode != HttpStatusCode.Unauthorized)
throw new Exception($"Expected 401 for anonymous create batch, got {anonRes.StatusCode}");
// 2. Teacher -> 403
var tToken = factory.GenerateToken("usr_u21_t", "teacher_u21", "Teacher", tenantId);
var tReq = new HttpRequestMessage(HttpMethod.Post, postUrl) { Content = JsonContent.Create(reqPayload) };
tReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tToken);
var tRes = await client.SendAsync(tReq);
if (tRes.StatusCode != HttpStatusCode.Forbidden)
throw new Exception($"Expected 403 for Teacher create batch, got {tRes.StatusCode}");
// 3. SchoolAdmin -> 403
var saToken = factory.GenerateToken("usr_u21_sa", "schooladmin_u21", "SchoolAdmin", tenantId);
var saReq = new HttpRequestMessage(HttpMethod.Post, postUrl) { Content = JsonContent.Create(reqPayload) };
saReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", saToken);
var saRes = await client.SendAsync(saReq);
if (saRes.StatusCode != HttpStatusCode.Forbidden)
throw new Exception($"Expected 403 for SchoolAdmin create batch, got {saRes.StatusCode}");
// 4. SuperAdmin -> 202 Accepted
var superToken = factory.GenerateToken("usr_u21_super", "superadmin_u21", "SuperAdmin", "global");
var superReq = new HttpRequestMessage(HttpMethod.Post, postUrl) { Content = JsonContent.Create(reqPayload) };
superReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var superRes = await client.SendAsync(superReq);
if (superRes.StatusCode != HttpStatusCode.Accepted)
throw new Exception($"Expected 202 for SuperAdmin create batch, got {superRes.StatusCode}");
var batchDto = await superRes.Content.ReadFromJsonAsync();
if (batchDto == null || string.IsNullOrWhiteSpace(batchDto.BatchId))
throw new Exception("Expected valid UpdateBatchDto returned from batch creation.");
// 5. Teacher 查询批次 -> 403
var tGetReq = new HttpRequestMessage(HttpMethod.Get, $"/api/admin/update-batches/{batchDto.BatchId}");
tGetReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tToken);
var tGetRes = await client.SendAsync(tGetReq);
if (tGetRes.StatusCode != HttpStatusCode.Forbidden)
throw new Exception($"Expected 403 for Teacher get batch, got {tGetRes.StatusCode}");
// 6. SuperAdmin 查询版本下拉 -> 200
var relReq = new HttpRequestMessage(HttpMethod.Get, "/api/admin/update-releases");
relReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var relRes = await client.SendAsync(relReq);
if (relRes.StatusCode != HttpStatusCode.OK)
throw new Exception($"Expected 200 for SuperAdmin get releases, got {relRes.StatusCode}");
}
///
/// U22: Batch Scope & Boundary Enforcement
/// 灰度批次严格限定设备规模 1 至 5 台;
/// 超过 5 台、空列表、重复设备、未指定 Canary、跨租户混合设备一律拒绝 (400 InvalidBatchScope)
///
private static async Task Test_U22_BatchScopeEnforcement()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u22_super", "superadmin", "SuperAdmin", "global");
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u22_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, "t_u22_a", "School A", new List { "d_a1", "d_a2", "d_a3", "d_a4", "d_a5", "d_a6" });
await SeedTenantAndDevicesAsync(db, "t_u22_b", "School B", new List { "d_b1" });
await SeedReleaseAsync(db, "rel_u22", "2.6.0");
}
async Task AssertBadRequest(CreateUpdateBatchRequest req, string expectedError)
{
var httpReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(req)
};
httpReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var res = await client.SendAsync(httpReq);
if (res.StatusCode != HttpStatusCode.BadRequest)
throw new Exception($"Expected 400 BadRequest, got {res.StatusCode}");
var body = await res.Content.ReadAsStringAsync();
if (!body.Contains(expectedError))
throw new Exception($"Expected error code '{expectedError}', got '{body}'");
}
// 1. 超过 5 台 (6台) -> 拒绝
await AssertBadRequest(new CreateUpdateBatchRequest
{
TenantId = "t_u22_a",
ReleaseId = "rel_u22",
CanaryDeviceId = "d_a1",
DeviceIds = new List { "d_a1", "d_a2", "d_a3", "d_a4", "d_a5", "d_a6" }
}, UpdateErrorCodes.InvalidBatchScope);
// 2. 空列表 (0台) -> 拒绝
await AssertBadRequest(new CreateUpdateBatchRequest
{
TenantId = "t_u22_a",
ReleaseId = "rel_u22",
CanaryDeviceId = "d_a1",
DeviceIds = new List()
}, UpdateErrorCodes.InvalidBatchScope);
// 3. 包含重复设备标识 -> 拒绝
await AssertBadRequest(new CreateUpdateBatchRequest
{
TenantId = "t_u22_a",
ReleaseId = "rel_u22",
CanaryDeviceId = "d_a1",
DeviceIds = new List { "d_a1", "d_a1" }
}, UpdateErrorCodes.InvalidBatchScope);
// 4. Canary 不在设备列表中 -> 拒绝
await AssertBadRequest(new CreateUpdateBatchRequest
{
TenantId = "t_u22_a",
ReleaseId = "rel_u22",
CanaryDeviceId = "d_a99_nonexistent",
DeviceIds = new List { "d_a1", "d_a2" }
}, UpdateErrorCodes.InvalidBatchScope);
// 5. 跨租户混合设备 (d_a1 属于 School A, d_b1 属于 School B) -> 拒绝
await AssertBadRequest(new CreateUpdateBatchRequest
{
TenantId = "t_u22_a",
ReleaseId = "rel_u22",
CanaryDeviceId = "d_a1",
DeviceIds = new List { "d_a1", "d_b1" }
}, UpdateErrorCodes.InvalidBatchScope);
// 6. 合法 1 台设备 (Canary 本身) -> 接受
var req1 = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = "t_u22_a",
ReleaseId = "rel_u22",
CanaryDeviceId = "d_a1",
DeviceIds = new List { "d_a1" }
})
};
req1.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var res1 = await client.SendAsync(req1);
if (res1.StatusCode != HttpStatusCode.Accepted)
throw new Exception($"Expected 202 for 1 device batch, got {res1.StatusCode}");
}
///
/// U23: Batch Idempotency & Repeat Protection
/// 携带相同 ClientRequestId 的批次创建请求返回相同批次 DTO (202 Accepted);
/// 数据库中严格仅存在 1 条批次记录与 1 条审计日志;
/// 存在正在升级设备时禁止创建新批次 (409 Conflict)
///
private static async Task Test_U23_BatchIdempotency()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u23_super", "superadmin", "SuperAdmin", "global");
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u23_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, "t_u23", "School U23", new List { "d_u23_1", "d_u23_2" });
await SeedReleaseAsync(db, "rel_u23", "2.6.0");
}
var clientReqId = "req_u23_client_idem_001";
var payload = new CreateUpdateBatchRequest
{
TenantId = "t_u23",
ReleaseId = "rel_u23",
CanaryDeviceId = "d_u23_1",
DeviceIds = new List { "d_u23_1", "d_u23_2" },
ClientRequestId = clientReqId
};
// 1. 第一次请求 -> 202
var req1 = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(payload) };
req1.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var res1 = await client.SendAsync(req1);
if (res1.StatusCode != HttpStatusCode.Accepted)
throw new Exception($"Expected 202, got {res1.StatusCode}");
var dto1 = await res1.Content.ReadFromJsonAsync();
// 2. 第二次请求相同 ClientRequestId -> 幂等 202 并返回相同 BatchId
var req2 = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(payload) };
req2.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var res2 = await client.SendAsync(req2);
if (res2.StatusCode != HttpStatusCode.Accepted)
throw new Exception($"Expected 202 for idempotent request, got {res2.StatusCode}");
var dto2 = await res2.Content.ReadFromJsonAsync();
if (dto1!.BatchId != dto2!.BatchId)
throw new Exception($"Expected same BatchId {dto1.BatchId}, got {dto2.BatchId}");
// 3. 校验数据库中仅有 1 个批次和 1 条创建审计记录
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var count = await db.DeviceUpdateBatches.CountAsync(b => b.ClientRequestId == clientReqId);
if (count != 1)
throw new Exception($"Expected exactly 1 batch in DB, found {count}");
var auditCount = await db.AdminAuditEvents.CountAsync(a => a.TargetId == dto1.BatchId && a.Action == "DEVICE_UPDATE_BATCH_CREATED");
if (auditCount != 1)
throw new Exception($"Expected exactly 1 audit event for batch, found {auditCount}");
}
// 4. 并发活跃冲突校验: 使用不同 ClientRequestId 但相同设备创建新批次 -> 409 Conflict
var payloadConflict = new CreateUpdateBatchRequest
{
TenantId = "t_u23",
ReleaseId = "rel_u23",
CanaryDeviceId = "d_u23_1",
DeviceIds = new List { "d_u23_1", "d_u23_2" },
ClientRequestId = "req_u23_diff_client"
};
var reqConflict = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches") { Content = JsonContent.Create(payloadConflict) };
reqConflict.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var resConflict = await client.SendAsync(reqConflict);
if (resConflict.StatusCode != HttpStatusCode.Conflict)
throw new Exception($"Expected 409 Conflict when devices have active updates, got {resConflict.StatusCode}");
}
///
/// U24: Canary Isolation & Dispatch Gate
/// 灰度批次创建后,首发金丝雀设备 DispatchGate 为 Released,其余 1~4 台物理置为 Held;
/// 后台调度 Worker 严格只拉取 Released 设备下发 StartDeviceUpdate 指令;
/// Held 设备物理接收 0 条指令并保持 PendingDispatch
///
private static async Task Test_U24_CanaryIsolation()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u24_super", "superadmin", "SuperAdmin", "global");
var tenantId = "t_u24";
var canaryId = "d_u24_c";
var otherIds = new List { "d_u24_2", "d_u24_3", "d_u24_4", "d_u24_5" };
var allIds = new List { canaryId }.Concat(otherIds).ToList();
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u24_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, tenantId, "School U24", allIds);
await SeedReleaseAsync(db, "rel_u24", "2.6.0");
}
// 创建 5 台设备批次
var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = tenantId,
ReleaseId = "rel_u24",
CanaryDeviceId = canaryId,
DeviceIds = allIds
})
};
createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var createRes = await client.SendAsync(createReq);
if (createRes.StatusCode != HttpStatusCode.Accepted)
throw new Exception($"Expected 202, got {createRes.StatusCode}");
var dto = await createRes.Content.ReadFromJsonAsync();
// 1. 验证创建时的门禁状态
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == dto!.BatchId).ToListAsync();
var canaryRun = runs.First(r => r.DeviceId == canaryId);
if (!canaryRun.IsCanary || canaryRun.DispatchGate != UpdateDispatchGates.Released)
throw new Exception($"Expected Canary run to have DispatchGate Released, got {canaryRun.DispatchGate}");
var heldRuns = runs.Where(r => r.DeviceId != canaryId).ToList();
if (heldRuns.Count != 4 || heldRuns.Any(r => r.DispatchGate != UpdateDispatchGates.Held))
throw new Exception("Expected all 4 non-canary runs to have DispatchGate Held.");
}
// 2. 触发调度 Worker (所有设备均在线)
await TriggerWorkerAsync(factory);
// 3. 验证物理隔离:仅 Canary 获得分发 (Downloading),其余 4 台物理接收 0 条指令 (保持 PendingDispatch 且 AttemptCount == 0)
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == dto!.BatchId).ToListAsync();
var canaryRun = runs.First(r => r.DeviceId == canaryId);
if (canaryRun.Status != UpdateStatuses.Downloading || canaryRun.DispatchedAtUtc == null)
throw new Exception($"Expected Canary run to be dispatched (Downloading), got {canaryRun.Status}");
var nonCanaries = runs.Where(r => r.DeviceId != canaryId).ToList();
foreach (var r in nonCanaries)
{
if (r.Status != UpdateStatuses.PendingDispatch || r.DispatchedAtUtc != null || r.DispatchAttemptCount != 0)
throw new Exception($"Physical isolation violated! Device {r.DeviceId} received dispatch: Status={r.Status}, AttemptCount={r.DispatchAttemptCount}");
}
}
}
///
/// U25: Canary Failure Automatic Batch Stop
/// 当金丝雀验证设备在升级过程中发生失败 (进入 RolledBack / Failed / RollbackFailed / TimedOut 终态);
/// 批次升级任务立即自动转入 Stopped 终态;
/// 其余 1~4 台受控设备继续物理保持 Held,绝对不接收任何更新指令
///
private static async Task Test_U25_CanaryFailureStop()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u25_super", "superadmin", "SuperAdmin", "global");
var tenantId = "t_u25";
var canaryId = "d_u25_canary";
var otherIds = new List { "d_u25_2", "d_u25_3", "d_u25_4" };
var allIds = new List { canaryId }.Concat(otherIds).ToList();
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u25_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, tenantId, "School U25", allIds);
await SeedReleaseAsync(db, "rel_u25", "2.6.0");
}
var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = tenantId,
ReleaseId = "rel_u25",
CanaryDeviceId = canaryId,
DeviceIds = allIds
})
};
createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var createRes = await client.SendAsync(createReq);
var batchDto = await createRes.Content.ReadFromJsonAsync();
// 1. Worker 分发 Canary
await TriggerWorkerAsync(factory);
string canaryRunId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var cRun = await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchDto!.BatchId && r.IsCanary);
canaryRunId = cRun.Id;
}
// 2. 金丝雀经历健康检查失败并执行自动回滚,上报 RolledBack
await AdvanceRunToRollbackAsync(client, canaryRunId, canaryId);
// 3. 验证批次状态自动变为 Stopped
var currentBatch = await GetBatchDtoAsync(client, batchDto!.BatchId, superToken);
if (currentBatch.Status != UpdateBatchStatuses.Stopped)
throw new Exception($"Expected batch status Stopped, got {currentBatch.Status}");
if (currentBatch.StoppedAtUtc == null)
throw new Exception("Expected StoppedAtUtc to be populated.");
// 4. 再次触发 Worker,验证其余 3 台设备依然保持 Held,未被下发
await TriggerWorkerAsync(factory);
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == batchDto.BatchId).ToListAsync();
var others = runs.Where(r => !r.IsCanary).ToList();
if (others.Any(r => r.Status != UpdateStatuses.PendingDispatch || r.DispatchGate != UpdateDispatchGates.Held))
throw new Exception("Held devices were improperly dispatched after canary failure!");
}
// 5. 对 Stopped 批次调用 continue -> 400 BatchInvalidState
var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/continue");
contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var contRes = await client.SendAsync(contReq);
if (contRes.StatusCode != HttpStatusCode.BadRequest)
throw new Exception($"Expected 400 for continue on stopped batch, got {contRes.StatusCode}");
}
///
/// U26: Canary Succeeded & Manual Continue
/// 金丝雀成功升级后,批次状态自动转入 AwaitingContinue;
/// 严禁自动继续,必须等待 SuperAdmin 人工点击放行 (POST /api/admin/update-batches/{id}/continue);
/// 放行后 Held 设备原子翻转为 Released,批次转为 RollingOut 并自动继续分发
///
private static async Task Test_U26_ManualContinue()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u26_super", "superadmin", "SuperAdmin", "global");
var tenantId = "t_u26";
var canaryId = "d_u26_c";
var otherIds = new List { "d_u26_2", "d_u26_3" };
var allIds = new List { canaryId }.Concat(otherIds).ToList();
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u26_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, tenantId, "School U26", allIds);
await SeedReleaseAsync(db, "rel_u26", "2.6.0");
}
var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = tenantId,
ReleaseId = "rel_u26",
CanaryDeviceId = canaryId,
DeviceIds = allIds
})
};
createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var createRes = await client.SendAsync(createReq);
var batchDto = await createRes.Content.ReadFromJsonAsync();
// 1. Canary 执行升级并上报 Succeeded
await TriggerWorkerAsync(factory);
string canaryRunId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var cRun = await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchDto!.BatchId && r.IsCanary);
canaryRunId = cRun.Id;
}
await AdvanceRunToSuccessAsync(client, canaryRunId, canaryId);
// 2. 验证批次状态变为 AwaitingContinue
var currentBatch = await GetBatchDtoAsync(client, batchDto!.BatchId, superToken);
if (currentBatch.Status != UpdateBatchStatuses.AwaitingContinue)
throw new Exception($"Expected batch status AwaitingContinue, got {currentBatch.Status}");
// 验证其余设备依然是 Held (零自动推进)
if (currentBatch.Summary.Held != 2 || currentBatch.Summary.Succeeded != 1)
throw new Exception($"Expected Held=2, Succeeded=1, got Held={currentBatch.Summary.Held}, Succeeded={currentBatch.Summary.Succeeded}");
// 3. SuperAdmin 人工点击放行
var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/continue");
contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var contRes = await client.SendAsync(contReq);
if (contRes.StatusCode != HttpStatusCode.OK)
throw new Exception($"Expected 200 OK for continue batch, got {contRes.StatusCode}");
var afterContBatch = await contRes.Content.ReadFromJsonAsync();
if (afterContBatch!.Status != UpdateBatchStatuses.RollingOut)
throw new Exception($"Expected batch status RollingOut, got {afterContBatch.Status}");
// 4. 验证 Held 设备门禁全部转为 Released
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == batchDto.BatchId && !r.IsCanary).ToListAsync();
if (runs.Any(r => r.DispatchGate != UpdateDispatchGates.Released))
throw new Exception("Not all held devices were flipped to Released!");
// 验证审计日志
var audit = await db.AdminAuditEvents.FirstOrDefaultAsync(a => a.TargetId == batchDto.BatchId && a.Action == "DEVICE_UPDATE_BATCH_CONTINUED");
if (audit == null) throw new Exception("Continue audit event missing.");
}
// 5. 触发 Worker,验证 2 台新放行设备立即开始分发 (Downloading)
await TriggerWorkerAsync(factory);
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var runs = await db.DeviceUpdateRuns.Where(r => r.BatchId == batchDto.BatchId && !r.IsCanary).ToListAsync();
if (runs.Any(r => r.Status != UpdateStatuses.Downloading))
throw new Exception("Released runs were not dispatched by worker!");
}
}
///
/// U27: Stop Safety & In-Flight Protection
/// 管理员人工停止批次 (POST /api/admin/update-batches/{id}/stop);
/// Held 状态的任务物理保持阻断;
/// 已经在执行中 (Downloading, Installing, HealthChecking) 的任务安全放行直至终态,绝不粗暴杀死
///
private static async Task Test_U27_StopSafety()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u27_super", "superadmin", "SuperAdmin", "global");
var tenantId = "t_u27";
var canaryId = "d_u27_c";
var devInFlight = "d_u27_flight";
var devHeld = "d_u27_held";
var allIds = new List { canaryId, devInFlight, devHeld };
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u27_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, tenantId, "School U27", allIds);
await SeedReleaseAsync(db, "rel_u27", "2.6.0");
}
// 1. 创建批次并完成 Canary 放行
var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = tenantId,
ReleaseId = "rel_u27",
CanaryDeviceId = canaryId,
DeviceIds = allIds
})
};
createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var createRes = await client.SendAsync(createReq);
var batchDto = await createRes.Content.ReadFromJsonAsync();
// Canary 成功
await TriggerWorkerAsync(factory);
string canaryRunId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
canaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchDto!.BatchId && r.IsCanary)).Id;
}
await AdvanceRunToSuccessAsync(client, canaryRunId, canaryId);
// 放行
var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto!.BatchId}/continue");
contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
await client.SendAsync(contReq);
// 调度两台放行设备,并将其中一台推进到 Installing
await TriggerWorkerAsync(factory);
string flightRunId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
flightRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.DeviceId == devInFlight)).Id;
var heldRun = await db.DeviceUpdateRuns.FirstAsync(r => r.DeviceId == devHeld);
// 模拟将 devHeld 恢复为 Held (以测试 Held 保护)
heldRun.DispatchGate = UpdateDispatchGates.Held;
heldRun.Status = UpdateStatuses.PendingDispatch;
await db.SaveChangesAsync();
}
var repVer = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status")
{
Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.Verified })
};
repVer.Headers.Add("X-Device-Id", devInFlight);
repVer.Headers.Add("X-Device-Token", $"token_{devInFlight}");
await client.SendAsync(repVer);
var repInst = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status")
{
Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.Installing })
};
repInst.Headers.Add("X-Device-Id", devInFlight);
repInst.Headers.Add("X-Device-Token", $"token_{devInFlight}");
await client.SendAsync(repInst);
// 2. 管理员调用 StopBatch
var stopReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/stop");
stopReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var stopRes = await client.SendAsync(stopReq);
if (stopRes.StatusCode != HttpStatusCode.OK)
throw new Exception($"Expected 200 OK for stop batch, got {stopRes.StatusCode}");
var stoppedDto = await stopRes.Content.ReadFromJsonAsync();
if (stoppedDto!.Status != UpdateBatchStatuses.Stopped)
throw new Exception($"Expected status Stopped, got {stoppedDto.Status}");
// 3. 验证安全门禁:执行中的设备允许继续上报并正常走向 Succeeded 终态
var repHc = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status")
{
Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.HealthChecking })
};
repHc.Headers.Add("X-Device-Id", devInFlight);
repHc.Headers.Add("X-Device-Token", $"token_{devInFlight}");
await client.SendAsync(repHc);
var reportFinish = new HttpRequestMessage(HttpMethod.Post, $"/api/device/update-runs/{flightRunId}/status")
{
Content = JsonContent.Create(new DeviceUpdateReportRequest { Status = UpdateStatuses.Succeeded })
};
reportFinish.Headers.Add("X-Device-Id", devInFlight);
reportFinish.Headers.Add("X-Device-Token", $"token_{devInFlight}");
var reportFinishRes = await client.SendAsync(reportFinish);
if (reportFinishRes.StatusCode != HttpStatusCode.OK)
throw new Exception("In-flight update was improperly rejected after batch stop!");
// 4. 重复调用 StopBatch 是幂等的
var stopReq2 = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchDto.BatchId}/stop");
stopReq2.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superToken);
var stopRes2 = await client.SendAsync(stopReq2);
if (stopRes2.StatusCode != HttpStatusCode.OK)
throw new Exception("Second stop request was not handled idempotently.");
}
///
/// U28: Mixed Outcome Truthfulness & Dynamic Summary Invariant
/// 灰度批次汇总数据严禁读取批次表持久化计数;
/// 必须基于 DeviceUpdateRun 实时动态聚合;
/// 严格守恒断言:Target == Held + Updating + Succeeded + RolledBack + Failed + RollbackFailed + TimedOut;
/// RolledBack 严格独立统计,绝不混淆为 Succeeded
///
private static async Task Test_U28_DynamicSummaryInvariant()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u28_super", "superadmin", "SuperAdmin", "global");
var tenantId = "t_u28";
var devCanary = "d_u28_c";
var devRb = "d_u28_rb";
var devFail = "d_u28_fail";
var devTimeout = "d_u28_to";
var devHeld = "d_u28_held";
var allIds = new List { devCanary, devRb, devFail, devTimeout, devHeld };
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u28_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, tenantId, "School U28", allIds);
var rel = await SeedReleaseAsync(db, "rel_u28", "2.6.0");
var now = DateTimeOffset.UtcNow;
var batch = new DeviceUpdateBatch
{
Id = "batch_u28_mixed",
TenantId = tenantId,
ReleaseId = rel.Id,
CanaryDeviceId = devCanary,
RequestedByUserId = "usr_u28_super",
Status = UpdateBatchStatuses.RollingOut,
TargetDeviceCount = 5,
CreatedAtUtc = now
};
var runs = new List
{
new() { Id = "r1", TenantId = tenantId, DeviceId = devCanary, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = true, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.Succeeded, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) },
new() { Id = "r2", TenantId = tenantId, DeviceId = devRb, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.RolledBack, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) },
new() { Id = "r3", TenantId = tenantId, DeviceId = devFail, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.Failed, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) },
new() { Id = "r4", TenantId = tenantId, DeviceId = devTimeout, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Released, Status = UpdateStatuses.TimedOut, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) },
new() { Id = "r5", TenantId = tenantId, DeviceId = devHeld, ReleaseId = rel.Id, BatchId = batch.Id, IsCanary = false, DispatchGate = UpdateDispatchGates.Held, Status = UpdateStatuses.PendingDispatch, TargetVersion = "2.6.0", CreatedAtUtc = now, DeadlineAtUtc = now.AddMinutes(5) }
};
db.DeviceUpdateBatches.Add(batch);
db.DeviceUpdateRuns.AddRange(runs);
await db.SaveChangesAsync();
}
var dto = await GetBatchDtoAsync(client, "batch_u28_mixed", superToken);
var s = dto.Summary;
if (s.Target != 5) throw new Exception($"Expected Target=5, got {s.Target}");
if (s.Succeeded != 1) throw new Exception($"Expected Succeeded=1, got {s.Succeeded}");
if (s.RolledBack != 1) throw new Exception($"Expected RolledBack=1, got {s.RolledBack}");
if (s.Failed != 1) throw new Exception($"Expected Failed=1, got {s.Failed}");
if (s.TimedOut != 1) throw new Exception($"Expected TimedOut=1, got {s.TimedOut}");
if (s.Held != 1) throw new Exception($"Expected Held=1, got {s.Held}");
if (s.Updating != 0) throw new Exception($"Expected Updating=0, got {s.Updating}");
// 验证守恒
int sum = s.Held + s.Updating + s.Succeeded + s.RolledBack + s.Failed + s.RollbackFailed + s.TimedOut;
if (s.Target != sum)
throw new Exception($"Summary invariant violated! Target={s.Target}, Sum={sum}");
}
///
/// U29: Server Restart & State Durability
/// 模拟服务器意外崩溃重启:
/// 批次状态、Canary 执行结果、未放行设备的 Held 门禁、审计日志等由数据库持久化保证;
/// 重启后新服务实例无缝恢复 AwaitingContinue 状态并成功继续放行
///
private static async Task Test_U29_RestartDurability()
{
var sharedDbName = "classnotify_restart_test_" + Guid.NewGuid().ToString("N");
var tenantId = "t_u29";
var canaryId = "d_u29_c";
var dev2 = "d_u29_2";
string batchId;
// 1. 实例 A 创建批次,Canary 完成并进入 AwaitingContinue
using (var factoryA = new CustomWebApplicationFactory(sharedDbName))
{
var clientA = factoryA.CreateClient();
var superTokenA = factoryA.GenerateToken("usr_u29_super", "superadmin", "SuperAdmin", "global");
using (var scope = factoryA.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u29_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, tenantId, "School U29", new List { canaryId, dev2 });
await SeedReleaseAsync(db, "rel_u29", "2.6.0");
}
var createReq = new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = tenantId,
ReleaseId = "rel_u29",
CanaryDeviceId = canaryId,
DeviceIds = new List { canaryId, dev2 }
})
};
createReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superTokenA);
var createRes = await clientA.SendAsync(createReq);
var batchDto = await createRes.Content.ReadFromJsonAsync();
batchId = batchDto!.BatchId;
await TriggerWorkerAsync(factoryA);
string canaryRunId;
using (var scope = factoryA.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
canaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == batchId && r.IsCanary)).Id;
}
await AdvanceRunToSuccessAsync(clientA, canaryRunId, canaryId);
// 实例 A 停止运行
}
// 2. 模拟服务器重启:启动全新实例 B 挂载相同数据库
using (var factoryB = new CustomWebApplicationFactory(sharedDbName))
{
var clientB = factoryB.CreateClient();
var superTokenB = factoryB.GenerateToken("usr_u29_super", "superadmin", "SuperAdmin", "global");
// 查询批次状态 -> 完美保留 AwaitingContinue
var batchDtoB = await GetBatchDtoAsync(clientB, batchId, superTokenB);
if (batchDtoB.Status != UpdateBatchStatuses.AwaitingContinue)
throw new Exception($"Expected durable status AwaitingContinue, got {batchDtoB.Status}");
if (batchDtoB.Summary.Held != 1 || batchDtoB.Summary.Succeeded != 1)
throw new Exception($"Durable summary mismatch: Held={batchDtoB.Summary.Held}, Succeeded={batchDtoB.Summary.Succeeded}");
// 在重启后的实例 B 上继续执行人工放行 -> 成功
var contReq = new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{batchId}/continue");
contReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", superTokenB);
var contRes = await clientB.SendAsync(contReq);
if (contRes.StatusCode != HttpStatusCode.OK)
throw new Exception($"Expected 200 on continued batch after restart, got {contRes.StatusCode}");
var continuedDto = await contRes.Content.ReadFromJsonAsync();
if (continuedDto!.Status != UpdateBatchStatuses.RollingOut)
throw new Exception($"Expected RollingOut status after restart, got {continuedDto.Status}");
}
}
///
/// U30: Full End-to-End Regression & Dual Scenarios
/// 完整执行并验证两大核心业务场景:
/// Scenario 1 (Good Release Rollout): 5 台设备,1 Canary 升级成功 -> AwaitingContinue -> 人工放行 -> 4 台全部成功 -> Completed
/// Scenario 2 (Bad Canary Stop): 5 台设备,Canary 验证遇到故障自动回滚 (RolledBack) -> 批次立即自动 Stopped -> 4 台设备物理未触碰 (Held=4, Succeeded=0)
///
private static async Task Test_U30_FullDualScenarios()
{
using var factory = new CustomWebApplicationFactory();
var client = factory.CreateClient();
var superToken = factory.GenerateToken("usr_u30_super", "superadmin", "SuperAdmin", "global");
// --------------------------------------------------------------------------------
// SCENARIO 1: 正常版本小规模灰度全流程演示 (Good Release Rollout)
// --------------------------------------------------------------------------------
Console.WriteLine("\n [Scenario 1: Good Release Staged Rollout]");
var t1 = "t_u30_s1";
var c1 = "d_s1_canary";
var s1Devs = new List { c1, "d_s1_2", "d_s1_3", "d_s1_4", "d_s1_5" };
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u30_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, t1, "Good Rollout School", s1Devs, "2.5.0");
await SeedReleaseAsync(db, "rel_v260_good", "2.6.0");
}
// 1. 创建 5 台设备批次
var s1CreateRes = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = t1,
ReleaseId = "rel_v260_good",
CanaryDeviceId = c1,
DeviceIds = s1Devs
}),
Headers = { Authorization = new AuthenticationHeaderValue("Bearer", superToken) }
});
if (s1CreateRes.StatusCode != HttpStatusCode.Accepted)
throw new Exception("Scenario 1 batch creation failed.");
var s1Batch = await s1CreateRes.Content.ReadFromJsonAsync();
// 2. 调度金丝雀设备
await TriggerWorkerAsync(factory);
string s1CanaryRunId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
s1CanaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == s1Batch!.BatchId && r.IsCanary)).Id;
}
// 3. 金丝雀全流程成功
await AdvanceRunToSuccessAsync(client, s1CanaryRunId, c1);
// 4. 验证批次进入 AwaitingContinue,其余 4 台处于 Held
var s1Mid = await GetBatchDtoAsync(client, s1Batch!.BatchId, superToken);
if (s1Mid.Status != UpdateBatchStatuses.AwaitingContinue || s1Mid.Summary.Held != 4 || s1Mid.Summary.Succeeded != 1)
throw new Exception($"Scenario 1 Canary did not gate into AwaitingContinue: Status={s1Mid.Status}, Held={s1Mid.Summary.Held}, Succeeded={s1Mid.Summary.Succeeded}");
// 5. 人工确认放行
var s1Cont = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, $"/api/admin/update-batches/{s1Batch.BatchId}/continue")
{
Headers = { Authorization = new AuthenticationHeaderValue("Bearer", superToken) }
});
if (s1Cont.StatusCode != HttpStatusCode.OK) throw new Exception("Scenario 1 continue failed.");
// 6. 调度放行后的 4 台设备
await TriggerWorkerAsync(factory);
List remainingRunIds;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
remainingRunIds = await db.DeviceUpdateRuns
.Where(r => r.BatchId == s1Batch.BatchId && !r.IsCanary)
.Select(r => r.Id)
.ToListAsync();
}
// 7. 4 台设备全部升级成功
foreach (var runId in remainingRunIds)
{
string devId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
devId = (await db.DeviceUpdateRuns.FirstAsync(r => r.Id == runId)).DeviceId;
}
await AdvanceRunToSuccessAsync(client, runId, devId);
}
// 8. 触发调度协调,批次最终转入 Completed
await TriggerWorkerAsync(factory);
var s1Final = await GetBatchDtoAsync(client, s1Batch.BatchId, superToken);
if (s1Final.Status != UpdateBatchStatuses.Completed)
throw new Exception($"Expected Scenario 1 batch Completed, got {s1Final.Status}");
if (s1Final.Summary.Target != 5 || s1Final.Summary.Succeeded != 5 || s1Final.Summary.Held != 0)
throw new Exception($"Expected all 5 Succeeded, got Succeeded={s1Final.Summary.Succeeded}, Held={s1Final.Summary.Held}");
Console.WriteLine(" -> Scenario 1 PASSED: 5 devices upgraded to 2.6.0, Batch: Completed.");
// --------------------------------------------------------------------------------
// SCENARIO 2: 坏版金丝雀自动回滚熔断停机全流程演示 (Bad Canary Automatic Rollback & Stop)
// --------------------------------------------------------------------------------
Console.WriteLine("\n [Scenario 2: Bad Canary Automatic Rollback & Safe Batch Stop]");
var t2 = "t_u30_s2";
var c2 = "d_s2_canary";
var s2Devs = new List { c2, "d_s2_2", "d_s2_3", "d_s2_4", "d_s2_5" };
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
await EnsureUserAsync(db, "usr_u30_super", "superadmin", "SuperAdmin", "global");
await SeedTenantAndDevicesAsync(db, t2, "Fault Rollback School", s2Devs, "2.5.0");
await SeedReleaseAsync(db, "rel_v270_bad", "2.7.0");
}
// 1. 创建 5 台设备批次
var s2CreateRes = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, "/api/admin/update-batches")
{
Content = JsonContent.Create(new CreateUpdateBatchRequest
{
TenantId = t2,
ReleaseId = "rel_v270_bad",
CanaryDeviceId = c2,
DeviceIds = s2Devs
}),
Headers = { Authorization = new AuthenticationHeaderValue("Bearer", superToken) }
});
if (s2CreateRes.StatusCode != HttpStatusCode.Accepted)
throw new Exception("Scenario 2 batch creation failed.");
var s2Batch = await s2CreateRes.Content.ReadFromJsonAsync();
// 2. 调度金丝雀设备
await TriggerWorkerAsync(factory);
string s2CanaryRunId;
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
s2CanaryRunId = (await db.DeviceUpdateRuns.FirstAsync(r => r.BatchId == s2Batch!.BatchId && r.IsCanary)).Id;
}
// 3. 金丝雀安装后健康检查失败,自愈回滚并上报 RolledBack
await AdvanceRunToRollbackAsync(client, s2CanaryRunId, c2);
// 4. 验证批次自动转为 Stopped,其余 4 台物理未触碰 (Held=4, Succeeded=0, 接收 0 条指令)
var s2Final = await GetBatchDtoAsync(client, s2Batch!.BatchId, superToken);
if (s2Final.Status != UpdateBatchStatuses.Stopped)
throw new Exception($"Expected Scenario 2 batch Stopped, got {s2Final.Status}");
if (s2Final.Summary.Target != 5 || s2Final.Summary.Held != 4 || s2Final.Summary.RolledBack != 1 || s2Final.Summary.Succeeded != 0)
throw new Exception($"Scenario 2 summary violation: Target={s2Final.Summary.Target}, Held={s2Final.Summary.Held}, RolledBack={s2Final.Summary.RolledBack}, Succeeded={s2Final.Summary.Succeeded}");
// 5. 再次触发 Worker,确认未放行设备保持 0 分发
await TriggerWorkerAsync(factory);
using (var scope = factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService();
var nonCanaries = await db.DeviceUpdateRuns.Where(r => r.BatchId == s2Batch.BatchId && !r.IsCanary).ToListAsync();
if (nonCanaries.Any(r => r.Status != UpdateStatuses.PendingDispatch || r.DispatchGate != UpdateDispatchGates.Held))
throw new Exception("Non-canary devices received update after canary rollback stop!");
}
Console.WriteLine(" -> Scenario 2 PASSED: Canary safely rolled back to 2.5.0, Batch: Stopped, 4 devices Held with 0 updates.");
}
#endregion
}