feat: Implement provisioning pipelines for subscription management
- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes. - Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging. - Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR. - Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes. - Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots. - Introduce XiboFeatureManifests for hardcoded feature ACLs per role. - Add docker-compose.dev.yml for local development with PostgreSQL setup.
This commit is contained in:
Matt Batchelder
106
OTSSignsOrchestrator.Server/Jobs/ByoiCertExpiryJob.cs
Normal file
106
OTSSignsOrchestrator.Server/Jobs/ByoiCertExpiryJob.cs
Normal file
@@ -0,0 +1,106 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Quartz;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Quartz job that runs daily to check BYOI certificate expiry dates across all enabled
|
||||
/// ByoiConfig entries. Alerts at 60, 30, and 7 day thresholds via FleetHub and logs to AuditLog.
|
||||
///
|
||||
/// Severity escalation:
|
||||
/// - > 7 days remaining → "Warning"
|
||||
/// - ≤ 7 days remaining → "Critical"
|
||||
/// </summary>
|
||||
// IMMUTABLE AuditLog — this job only appends, never updates or deletes audit records.
|
||||
[DisallowConcurrentExecution]
|
||||
public sealed class ByoiCertExpiryJob : IJob
|
||||
{
|
||||
/// <summary>Alert thresholds in days. Alerts fire when remaining days ≤ threshold.</summary>
|
||||
internal static readonly int[] AlertThresholdDays = [60, 30, 7];
|
||||
|
||||
/// <summary>Days at or below which severity escalates to "Critical".</summary>
|
||||
internal const int CriticalThresholdDays = 7;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<ByoiCertExpiryJob> _logger;
|
||||
|
||||
public ByoiCertExpiryJob(
|
||||
IServiceProvider services,
|
||||
ILogger<ByoiCertExpiryJob> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Execute(IJobExecutionContext context)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
|
||||
var configs = await db.ByoiConfigs
|
||||
.AsNoTracking()
|
||||
.Include(b => b.Instance)
|
||||
.ThenInclude(i => i.Customer)
|
||||
.Where(b => b.Enabled)
|
||||
.ToListAsync(context.CancellationToken);
|
||||
|
||||
foreach (var config in configs)
|
||||
{
|
||||
var daysRemaining = (config.CertExpiry - DateTime.UtcNow).TotalDays;
|
||||
var abbrev = config.Instance.Customer.Abbreviation;
|
||||
|
||||
if (!ShouldAlert(daysRemaining))
|
||||
continue;
|
||||
|
||||
var severity = GetSeverity(daysRemaining);
|
||||
var daysInt = (int)Math.Floor(daysRemaining);
|
||||
var message = daysRemaining <= 0
|
||||
? $"BYOI cert for {abbrev} has EXPIRED."
|
||||
: $"BYOI cert for {abbrev} expires in {daysInt} days.";
|
||||
|
||||
_logger.LogWarning("BYOI cert expiry alert: {Severity} — {Message}", severity, message);
|
||||
|
||||
await hub.Clients.All.SendAlertRaised(severity, message);
|
||||
|
||||
// Append-only audit log
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = config.InstanceId,
|
||||
Actor = "ByoiCertExpiryJob",
|
||||
Action = "CertExpiryAlert",
|
||||
Target = config.Slug,
|
||||
Outcome = severity,
|
||||
Detail = message,
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
}
|
||||
|
||||
await db.SaveChangesAsync(context.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether an alert should fire based on remaining days.
|
||||
/// Alerts at ≤ 60, ≤ 30, ≤ 7 days (or already expired).
|
||||
/// </summary>
|
||||
internal static bool ShouldAlert(double daysRemaining)
|
||||
{
|
||||
foreach (var threshold in AlertThresholdDays)
|
||||
{
|
||||
if (daysRemaining <= threshold)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns "Critical" when ≤ 7 days remain, otherwise "Warning".
|
||||
/// </summary>
|
||||
internal static string GetSeverity(double daysRemaining) =>
|
||||
daysRemaining <= CriticalThresholdDays ? "Critical" : "Warning";
|
||||
}
|
||||
118
OTSSignsOrchestrator.Server/Jobs/DailySnapshotJob.cs
Normal file
118
OTSSignsOrchestrator.Server/Jobs/DailySnapshotJob.cs
Normal file
@@ -0,0 +1,118 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Quartz;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Quartz job scheduled at 2 AM UTC daily (<c>0 0 2 * * ?</c>).
|
||||
/// For each active <see cref="Instance"/>: calls <see cref="XiboApiClientExtensions.GetAllPagesAsync{T}"/>
|
||||
/// with <c>authorised=1</c> to count authorised displays; inserts a <see cref="ScreenSnapshot"/> row.
|
||||
///
|
||||
/// Uses ON CONFLICT DO NOTHING semantics to protect against double-runs.
|
||||
/// THIS DATA CANNOT BE RECOVERED — if the job misses a day, that data is permanently lost.
|
||||
/// </summary>
|
||||
[DisallowConcurrentExecution]
|
||||
public sealed class DailySnapshotJob : IJob
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<DailySnapshotJob> _logger;
|
||||
|
||||
public DailySnapshotJob(
|
||||
IServiceProvider services,
|
||||
ILogger<DailySnapshotJob> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Execute(IJobExecutionContext context)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var clientFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<Core.Services.SettingsService>();
|
||||
|
||||
var today = DateOnly.FromDateTime(DateTime.UtcNow);
|
||||
|
||||
var instances = await db.Instances
|
||||
.Include(i => i.Customer)
|
||||
.Include(i => i.OauthAppRegistries)
|
||||
.Where(i => i.Customer.Status == CustomerStatus.Active)
|
||||
.ToListAsync(context.CancellationToken);
|
||||
|
||||
_logger.LogInformation("DailySnapshotJob: processing {Count} active instance(s) for {Date}",
|
||||
instances.Count, today);
|
||||
|
||||
foreach (var instance in instances)
|
||||
{
|
||||
var abbrev = instance.Customer.Abbreviation;
|
||||
try
|
||||
{
|
||||
var oauthApp = instance.OauthAppRegistries.FirstOrDefault();
|
||||
if (oauthApp is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"DailySnapshotJob: skipping {Abbrev} — no OAuth app registered", abbrev);
|
||||
continue;
|
||||
}
|
||||
|
||||
var secret = await settings.GetAsync(
|
||||
Core.Services.SettingsService.InstanceOAuthSecretId(abbrev));
|
||||
if (string.IsNullOrEmpty(secret))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"DailySnapshotJob: skipping {Abbrev} — OAuth secret not found", abbrev);
|
||||
continue;
|
||||
}
|
||||
|
||||
var client = await clientFactory.CreateAsync(
|
||||
instance.XiboUrl, oauthApp.ClientId, secret);
|
||||
|
||||
var displays = await client.GetAllPagesAsync(
|
||||
(start, length) => client.GetDisplaysAsync(start, length, authorised: 1));
|
||||
|
||||
var screenCount = displays.Count;
|
||||
|
||||
// ON CONFLICT DO NOTHING — protect against double-runs.
|
||||
// Check if a snapshot already exists for this instance + date.
|
||||
var exists = await db.ScreenSnapshots.AnyAsync(
|
||||
s => s.InstanceId == instance.Id && s.SnapshotDate == today,
|
||||
context.CancellationToken);
|
||||
|
||||
if (!exists)
|
||||
{
|
||||
db.ScreenSnapshots.Add(new ScreenSnapshot
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = instance.Id,
|
||||
SnapshotDate = today,
|
||||
ScreenCount = screenCount,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
_logger.LogInformation(
|
||||
"DailySnapshotJob: {Abbrev} — {Count} authorised display(s)",
|
||||
abbrev, screenCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"DailySnapshotJob: {Abbrev} — snapshot already exists for {Date}, skipping",
|
||||
abbrev, today);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// THIS DATA CANNOT BE RECOVERED — log prominently
|
||||
_logger.LogWarning(ex,
|
||||
"DailySnapshotJob: FAILED to capture snapshot for {Abbrev} on {Date}. " +
|
||||
"This data is permanently lost.", abbrev, today);
|
||||
}
|
||||
}
|
||||
|
||||
await db.SaveChangesAsync(context.CancellationToken);
|
||||
}
|
||||
}
|
||||
175
OTSSignsOrchestrator.Server/Jobs/ScheduledReportJob.cs
Normal file
175
OTSSignsOrchestrator.Server/Jobs/ScheduledReportJob.cs
Normal file
@@ -0,0 +1,175 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Quartz;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Reports;
|
||||
using OTSSignsOrchestrator.Server.Services;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Quartz job with two triggers:
|
||||
/// - Weekly (Monday 08:00 UTC): fleet health PDF → operator email list
|
||||
/// - Monthly (1st of month 08:00 UTC): billing CSV + fleet health PDF → operators;
|
||||
/// per-customer usage PDF → each active customer's admin email
|
||||
/// </summary>
|
||||
[DisallowConcurrentExecution]
|
||||
public sealed class ScheduledReportJob : IJob
|
||||
{
|
||||
/// <summary>Quartz job data key indicating whether this is a monthly trigger.</summary>
|
||||
public const string IsMonthlyKey = "IsMonthly";
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<ScheduledReportJob> _logger;
|
||||
|
||||
public ScheduledReportJob(
|
||||
IServiceProvider services,
|
||||
ILogger<ScheduledReportJob> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Execute(IJobExecutionContext context)
|
||||
{
|
||||
var isMonthly = context.MergedJobDataMap.GetBoolean(IsMonthlyKey);
|
||||
|
||||
_logger.LogInformation("ScheduledReportJob fired — isMonthly={IsMonthly}", isMonthly);
|
||||
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var billing = scope.ServiceProvider.GetRequiredService<BillingReportService>();
|
||||
var pdfService = scope.ServiceProvider.GetRequiredService<FleetHealthPdfService>();
|
||||
var emailService = scope.ServiceProvider.GetRequiredService<EmailService>();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
|
||||
// Get operator email list (admin operators)
|
||||
var operatorEmails = await db.Operators
|
||||
.AsNoTracking()
|
||||
.Where(o => o.Role == OperatorRole.Admin)
|
||||
.Select(o => o.Email)
|
||||
.ToListAsync(context.CancellationToken);
|
||||
|
||||
if (operatorEmails.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("No admin operators found — skipping report email dispatch");
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Weekly: fleet health PDF for past 7 days ───────────────────────
|
||||
var to = DateOnly.FromDateTime(DateTime.UtcNow);
|
||||
var weekFrom = to.AddDays(-7);
|
||||
|
||||
byte[] fleetPdf;
|
||||
try
|
||||
{
|
||||
fleetPdf = await pdfService.GenerateFleetHealthPdfAsync(weekFrom, to);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to generate weekly fleet health PDF");
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var email in operatorEmails)
|
||||
{
|
||||
var success = await emailService.SendReportEmailAsync(
|
||||
email,
|
||||
$"fleet-health-{weekFrom:yyyy-MM-dd}-{to:yyyy-MM-dd}.pdf",
|
||||
fleetPdf,
|
||||
"application/pdf");
|
||||
|
||||
if (!success)
|
||||
_logger.LogError("Failed to send weekly fleet health PDF to {Email}", email);
|
||||
}
|
||||
|
||||
// ── Monthly: billing CSV + per-customer usage PDFs ─────────────────
|
||||
if (!isMonthly) return;
|
||||
|
||||
var monthStart = new DateOnly(to.Year, to.Month, 1).AddMonths(-1);
|
||||
var monthEnd = monthStart.AddMonths(1).AddDays(-1);
|
||||
|
||||
// Billing CSV
|
||||
byte[]? billingCsv = null;
|
||||
try
|
||||
{
|
||||
billingCsv = await billing.GenerateBillingCsvAsync(monthStart, monthEnd);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to generate monthly billing CSV");
|
||||
}
|
||||
|
||||
// Monthly fleet health PDF
|
||||
byte[]? monthlyFleetPdf = null;
|
||||
try
|
||||
{
|
||||
monthlyFleetPdf = await pdfService.GenerateFleetHealthPdfAsync(monthStart, monthEnd);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to generate monthly fleet health PDF");
|
||||
}
|
||||
|
||||
// Send monthly reports to operators
|
||||
foreach (var email in operatorEmails)
|
||||
{
|
||||
if (billingCsv is not null)
|
||||
{
|
||||
var ok = await emailService.SendReportEmailAsync(
|
||||
email,
|
||||
$"billing-{monthStart:yyyy-MM-dd}-{monthEnd:yyyy-MM-dd}.csv",
|
||||
billingCsv,
|
||||
"text/csv");
|
||||
if (!ok)
|
||||
_logger.LogError("Failed to send monthly billing CSV to {Email}", email);
|
||||
}
|
||||
|
||||
if (monthlyFleetPdf is not null)
|
||||
{
|
||||
var ok = await emailService.SendReportEmailAsync(
|
||||
email,
|
||||
$"fleet-health-{monthStart:yyyy-MM-dd}-{monthEnd:yyyy-MM-dd}.pdf",
|
||||
monthlyFleetPdf,
|
||||
"application/pdf");
|
||||
if (!ok)
|
||||
_logger.LogError("Failed to send monthly fleet health PDF to {Email}", email);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-customer usage PDFs → customer admin emails
|
||||
var activeCustomers = await db.Customers
|
||||
.AsNoTracking()
|
||||
.Where(c => c.Status == CustomerStatus.Active)
|
||||
.ToListAsync(context.CancellationToken);
|
||||
|
||||
foreach (var customer in activeCustomers)
|
||||
{
|
||||
try
|
||||
{
|
||||
var usagePdf = await pdfService.GenerateCustomerUsagePdfAsync(
|
||||
customer.Id, monthStart, monthEnd);
|
||||
|
||||
var ok = await emailService.SendReportEmailAsync(
|
||||
customer.AdminEmail,
|
||||
$"usage-{customer.Abbreviation}-{monthStart:yyyy-MM-dd}-{monthEnd:yyyy-MM-dd}.pdf",
|
||||
usagePdf,
|
||||
"application/pdf");
|
||||
|
||||
if (!ok)
|
||||
_logger.LogError("Failed to send usage PDF to {Email} for customer {Abbrev}",
|
||||
customer.AdminEmail, customer.Abbreviation);
|
||||
else
|
||||
_logger.LogInformation("Sent usage PDF to {Email} for customer {Abbrev}",
|
||||
customer.AdminEmail, customer.Abbreviation);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to generate/send usage PDF for customer {Abbrev}",
|
||||
customer.Abbreviation);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Monthly report dispatch complete — {CustomerCount} customer reports processed",
|
||||
activeCustomers.Count);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user