using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ClassNotify.Server.Data;
using ClassNotify.Server.Hubs;
using ClassNotify.Shared.Contracts;
using ClassNotify.Shared.Models;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ClassNotify.Server.Services;
///
/// 客户端在线升级后台分发调度器 (P2.5 RC1-4 Phase 1)
/// 严格基于数据库 PendingDispatch 状态作为可靠持久化队列,支持服务器重启后完全自动恢复
/// 离线真实等待重试,直至 Deadline 触发 TimedOut,绝不虚构状态
///
public sealed class DeviceUpdateDispatchWorker : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly IDeviceUpdateDispatchNotifier _notifier;
private readonly ILogger _logger;
public DeviceUpdateDispatchWorker(
IServiceScopeFactory scopeFactory,
IDeviceUpdateDispatchNotifier notifier,
ILogger logger)
{
_scopeFactory = scopeFactory;
_notifier = notifier;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("DeviceUpdateDispatchWorker started.");
while (!stoppingToken.IsCancellationRequested)
{
try
{
var processed = await ProcessPendingUpdatesAsync(stoppingToken);
if (!processed)
{
await Task.WhenAny(_notifier.WaitAsync(stoppingToken), Task.Delay(1000, stoppingToken));
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unexpected error in DeviceUpdateDispatchWorker loop");
await Task.Delay(1000, stoppingToken);
}
}
_logger.LogInformation("DeviceUpdateDispatchWorker stopped.");
}
private async Task ProcessPendingUpdatesAsync(CancellationToken ct)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService();
var presenceService = scope.ServiceProvider.GetRequiredService();
var hubContext = scope.ServiceProvider.GetRequiredService>();
var now = DateTimeOffset.UtcNow;
// 1. 扫描超时任务 (处于非终态且超过 DeadlineAtUtc)
var timedOutRuns = await db.DeviceUpdateRuns
.Where(u => UpdateStatuses.ActiveStatuses.Contains(u.Status) && u.DeadlineAtUtc <= now)
.Take(50)
.ToListAsync(ct);
if (timedOutRuns.Count > 0)
{
foreach (var r in timedOutRuns)
{
r.Status = UpdateStatuses.TimedOut;
r.ErrorCode ??= UpdateErrorCodes.UpdateTimeout;
r.ErrorMessage ??= "Update exceeded allowed execution deadline.";
r.CompletedAtUtc = now;
_logger.LogInformation("DeviceUpdateRun {Id} for device {DeviceId} timed out at {Now:u} (deadline was {Deadline:u})",
r.Id, r.DeviceId, now, r.DeadlineAtUtc);
}
await db.SaveChangesAsync(ct);
}
// 2. 扫描待分发任务 (Status == PendingDispatch && DispatchGate == Released && NextDispatchAttemptAtUtc <= now && DeadlineAtUtc > now)
// 关键门禁 U24: Held 任务绝对不处于分发队列,物理隔离未放行设备
var pendingRuns = await db.DeviceUpdateRuns
.Where(u => u.Status == UpdateStatuses.PendingDispatch
&& u.DispatchGate == UpdateDispatchGates.Released
&& (u.NextDispatchAttemptAtUtc == null || u.NextDispatchAttemptAtUtc <= now)
&& u.DeadlineAtUtc > now)
.OrderBy(u => u.CreatedAtUtc)
.Take(50)
.ToListAsync(ct);
foreach (var run in pendingRuns)
{
var device = await db.Devices.AsNoTracking().FirstOrDefaultAsync(d => d.Id == run.DeviceId, ct);
var release = await db.UpdateReleases.AsNoTracking().FirstOrDefaultAsync(r => r.Id == run.ReleaseId, ct);
if (release == null || !release.IsEnabled)
{
run.Status = UpdateStatuses.Failed;
run.ErrorCode = UpdateErrorCodes.ReleaseDisabled;
run.ErrorMessage = "Target release does not exist or has been disabled.";
run.CompletedAtUtc = now;
continue;
}
bool isOnline = device != null && presenceService.IsOnline(device.Id, device.LastHeartbeatAt);
if (isOnline)
{
// 设备在线:通过 tenant-qualified SignalR 下发 StartDeviceUpdateCommand (U4)
var command = new StartDeviceUpdateCommand
{
UpdateRunId = run.Id,
ReleaseId = release.Id,
CorrelationId = run.CorrelationId,
TargetVersion = release.Version,
ExpectedSha256 = release.PackageSha256,
ManifestSignature = release.ManifestSignature,
DeadlineAtUtc = run.DeadlineAtUtc
};
var tenantChannel = $"t:{run.TenantId}:d:{run.DeviceId}";
await hubContext.Clients.Group(tenantChannel).StartDeviceUpdate(command);
run.Status = UpdateStatuses.Downloading;
run.DispatchedAtUtc = now;
run.DispatchAttemptCount += 1;
run.LastDispatchAttemptAtUtc = now;
_logger.LogInformation("Successfully dispatched StartDeviceUpdateCommand to {Channel} for run {RunId} (Release={Release}, Version={Version}). State -> Downloading.",
tenantChannel, run.Id, release.Id, release.Version);
}
else
{
// 设备离线:保持 PendingDispatch,按重试梯级安排下一次检测 (U6)
run.DispatchAttemptCount += 1;
run.LastDispatchAttemptAtUtc = now;
int backoffSeconds = run.DispatchAttemptCount switch
{
1 => 2,
2 => 5,
3 => 10,
_ => 15
};
run.NextDispatchAttemptAtUtc = now.AddSeconds(backoffSeconds);
_logger.LogInformation("Device {DeviceId} for update run {RunId} is offline. Retry scheduled in {Seconds}s (Attempt={Attempt})",
run.DeviceId, run.Id, backoffSeconds, run.DispatchAttemptCount);
}
}
// 3. 自动协调活跃中的批次状态 (CanaryRunning -> AwaitingContinue / Stopped; RollingOut -> Completed)
await ReconcileBatchesAsync(db, now, ct);
await db.SaveChangesAsync(ct);
return pendingRuns.Count > 0 || timedOutRuns.Count > 0;
}
private async Task ReconcileBatchesAsync(ClassNotifyDbContext db, DateTimeOffset now, CancellationToken ct)
{
var activeBatches = await db.DeviceUpdateBatches
.Include(b => b.Runs)
.Where(b => b.Status == UpdateBatchStatuses.CanaryRunning || b.Status == UpdateBatchStatuses.RollingOut)
.ToListAsync(ct);
if (activeBatches.Count == 0) return;
foreach (var batch in activeBatches)
{
if (batch.Status == UpdateBatchStatuses.CanaryRunning)
{
var canaryRun = batch.Runs.FirstOrDefault(r => r.IsCanary);
if (canaryRun != null)
{
if (canaryRun.Status == UpdateStatuses.Succeeded)
{
batch.Status = UpdateBatchStatuses.AwaitingContinue;
_logger.LogInformation("Batch {BatchId} Canary succeeded. Status -> AwaitingContinue.", batch.Id);
}
else if (UpdateStatuses.TerminalStatuses.Contains(canaryRun.Status))
{
batch.Status = UpdateBatchStatuses.Stopped;
batch.StoppedAtUtc = now;
_logger.LogWarning("Batch {BatchId} Canary reached terminal failure {Status}. Batch -> Stopped.", batch.Id, canaryRun.Status);
}
}
}
else if (batch.Status == UpdateBatchStatuses.RollingOut)
{
bool allTerminal = batch.Runs.Count > 0 && batch.Runs.All(r => UpdateStatuses.TerminalStatuses.Contains(r.Status));
if (allTerminal)
{
batch.Status = UpdateBatchStatuses.Completed;
batch.CompletedAtUtc = now;
_logger.LogInformation("Batch {BatchId} all {Count} runs reached terminal states. Status -> Completed.", batch.Id, batch.Runs.Count);
}
}
}
}
}