using System; using System.Linq; using System.Threading; using System.Threading.Tasks; using ClassNotify.Server.Data; using ClassNotify.Server.Hubs; using ClassNotify.Shared.Contracts; using ClassNotify.Shared.Models; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace ClassNotify.Server.Services; /// /// 客户端在线升级后台分发调度器 (P2.5 RC1-4 Phase 1) /// 严格基于数据库 PendingDispatch 状态作为可靠持久化队列,支持服务器重启后完全自动恢复 /// 离线真实等待重试,直至 Deadline 触发 TimedOut,绝不虚构状态 /// public sealed class DeviceUpdateDispatchWorker : BackgroundService { private readonly IServiceScopeFactory _scopeFactory; private readonly IDeviceUpdateDispatchNotifier _notifier; private readonly ILogger _logger; public DeviceUpdateDispatchWorker( IServiceScopeFactory scopeFactory, IDeviceUpdateDispatchNotifier notifier, ILogger logger) { _scopeFactory = scopeFactory; _notifier = notifier; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation("DeviceUpdateDispatchWorker started."); while (!stoppingToken.IsCancellationRequested) { try { var processed = await ProcessPendingUpdatesAsync(stoppingToken); if (!processed) { await Task.WhenAny(_notifier.WaitAsync(stoppingToken), Task.Delay(1000, stoppingToken)); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } catch (Exception ex) { _logger.LogError(ex, "Unexpected error in DeviceUpdateDispatchWorker loop"); await Task.Delay(1000, stoppingToken); } } _logger.LogInformation("DeviceUpdateDispatchWorker stopped."); } private async Task ProcessPendingUpdatesAsync(CancellationToken ct) { using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var presenceService = scope.ServiceProvider.GetRequiredService(); var hubContext = scope.ServiceProvider.GetRequiredService>(); var now = DateTimeOffset.UtcNow; // 1. 扫描超时任务 (处于非终态且超过 DeadlineAtUtc) var timedOutRuns = await db.DeviceUpdateRuns .Where(u => UpdateStatuses.ActiveStatuses.Contains(u.Status) && u.DeadlineAtUtc <= now) .Take(50) .ToListAsync(ct); if (timedOutRuns.Count > 0) { foreach (var r in timedOutRuns) { r.Status = UpdateStatuses.TimedOut; r.ErrorCode ??= UpdateErrorCodes.UpdateTimeout; r.ErrorMessage ??= "Update exceeded allowed execution deadline."; r.CompletedAtUtc = now; _logger.LogInformation("DeviceUpdateRun {Id} for device {DeviceId} timed out at {Now:u} (deadline was {Deadline:u})", r.Id, r.DeviceId, now, r.DeadlineAtUtc); } await db.SaveChangesAsync(ct); } // 2. 扫描待分发任务 (Status == PendingDispatch && DispatchGate == Released && NextDispatchAttemptAtUtc <= now && DeadlineAtUtc > now) // 关键门禁 U24: Held 任务绝对不处于分发队列,物理隔离未放行设备 var pendingRuns = await db.DeviceUpdateRuns .Where(u => u.Status == UpdateStatuses.PendingDispatch && u.DispatchGate == UpdateDispatchGates.Released && (u.NextDispatchAttemptAtUtc == null || u.NextDispatchAttemptAtUtc <= now) && u.DeadlineAtUtc > now) .OrderBy(u => u.CreatedAtUtc) .Take(50) .ToListAsync(ct); foreach (var run in pendingRuns) { var device = await db.Devices.AsNoTracking().FirstOrDefaultAsync(d => d.Id == run.DeviceId, ct); var release = await db.UpdateReleases.AsNoTracking().FirstOrDefaultAsync(r => r.Id == run.ReleaseId, ct); if (release == null || !release.IsEnabled) { run.Status = UpdateStatuses.Failed; run.ErrorCode = UpdateErrorCodes.ReleaseDisabled; run.ErrorMessage = "Target release does not exist or has been disabled."; run.CompletedAtUtc = now; continue; } bool isOnline = device != null && presenceService.IsOnline(device.Id, device.LastHeartbeatAt); if (isOnline) { // 设备在线:通过 tenant-qualified SignalR 下发 StartDeviceUpdateCommand (U4) var command = new StartDeviceUpdateCommand { UpdateRunId = run.Id, ReleaseId = release.Id, CorrelationId = run.CorrelationId, TargetVersion = release.Version, ExpectedSha256 = release.PackageSha256, ManifestSignature = release.ManifestSignature, DeadlineAtUtc = run.DeadlineAtUtc }; var tenantChannel = $"t:{run.TenantId}:d:{run.DeviceId}"; await hubContext.Clients.Group(tenantChannel).StartDeviceUpdate(command); run.Status = UpdateStatuses.Downloading; run.DispatchedAtUtc = now; run.DispatchAttemptCount += 1; run.LastDispatchAttemptAtUtc = now; _logger.LogInformation("Successfully dispatched StartDeviceUpdateCommand to {Channel} for run {RunId} (Release={Release}, Version={Version}). State -> Downloading.", tenantChannel, run.Id, release.Id, release.Version); } else { // 设备离线:保持 PendingDispatch,按重试梯级安排下一次检测 (U6) run.DispatchAttemptCount += 1; run.LastDispatchAttemptAtUtc = now; int backoffSeconds = run.DispatchAttemptCount switch { 1 => 2, 2 => 5, 3 => 10, _ => 15 }; run.NextDispatchAttemptAtUtc = now.AddSeconds(backoffSeconds); _logger.LogInformation("Device {DeviceId} for update run {RunId} is offline. Retry scheduled in {Seconds}s (Attempt={Attempt})", run.DeviceId, run.Id, backoffSeconds, run.DispatchAttemptCount); } } // 3. 自动协调活跃中的批次状态 (CanaryRunning -> AwaitingContinue / Stopped; RollingOut -> Completed) await ReconcileBatchesAsync(db, now, ct); await db.SaveChangesAsync(ct); return pendingRuns.Count > 0 || timedOutRuns.Count > 0; } private async Task ReconcileBatchesAsync(ClassNotifyDbContext db, DateTimeOffset now, CancellationToken ct) { var activeBatches = await db.DeviceUpdateBatches .Include(b => b.Runs) .Where(b => b.Status == UpdateBatchStatuses.CanaryRunning || b.Status == UpdateBatchStatuses.RollingOut) .ToListAsync(ct); if (activeBatches.Count == 0) return; foreach (var batch in activeBatches) { if (batch.Status == UpdateBatchStatuses.CanaryRunning) { var canaryRun = batch.Runs.FirstOrDefault(r => r.IsCanary); if (canaryRun != null) { if (canaryRun.Status == UpdateStatuses.Succeeded) { batch.Status = UpdateBatchStatuses.AwaitingContinue; _logger.LogInformation("Batch {BatchId} Canary succeeded. Status -> AwaitingContinue.", batch.Id); } else if (UpdateStatuses.TerminalStatuses.Contains(canaryRun.Status)) { batch.Status = UpdateBatchStatuses.Stopped; batch.StoppedAtUtc = now; _logger.LogWarning("Batch {BatchId} Canary reached terminal failure {Status}. Batch -> Stopped.", batch.Id, canaryRun.Status); } } } else if (batch.Status == UpdateBatchStatuses.RollingOut) { bool allTerminal = batch.Runs.Count > 0 && batch.Runs.All(r => UpdateStatuses.TerminalStatuses.Contains(r.Status)); if (allTerminal) { batch.Status = UpdateBatchStatuses.Completed; batch.CompletedAtUtc = now; _logger.LogInformation("Batch {BatchId} all {Count} runs reached terminal states. Status -> Completed.", batch.Id, batch.Runs.Count); } } } } }