feat: Implement provisioning pipelines for subscription management
- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes. - Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging. - Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR. - Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes. - Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots. - Introduce XiboFeatureManifests for hardcoded feature ACLs per role. - Add docker-compose.dev.yml for local development with PostgreSQL setup.
This commit is contained in:
Matt Batchelder
195
OTSSignsOrchestrator.Server/Workers/SuspendPipeline.cs
Normal file
195
OTSSignsOrchestrator.Server/Workers/SuspendPipeline.cs
Normal file
@@ -0,0 +1,195 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Renci.SshNet;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Subscription suspension pipeline — scales down Docker services and marks status.
|
||||
/// Handles <c>JobType = "suspend"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. scale-down — SSH docker service scale web=0, xmr=0
|
||||
/// 2. update-status — Customer.Status = Suspended, Instance.HealthStatus = Degraded
|
||||
/// 3. audit-log — Append-only AuditLog entry
|
||||
/// 4. broadcast — InstanceStatusChanged via FleetHub
|
||||
/// </summary>
|
||||
public sealed class SuspendPipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "suspend";
|
||||
|
||||
private const int TotalSteps = 4;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<SuspendPipeline> _logger;
|
||||
|
||||
public SuspendPipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<SuspendPipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
|
||||
// ── Step 1: scale-down ──────────────────────────────────────────────
|
||||
await runner.RunAsync("scale-down", async () =>
|
||||
{
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
var cmd = $"docker service scale xibo-{abbrev}_web=0 xibo-{abbrev}_xmr=0";
|
||||
var result = RunSshCommand(sshClient, cmd);
|
||||
return $"Scaled down services for xibo-{abbrev}. Output: {result}";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: update-status ───────────────────────────────────────────
|
||||
await runner.RunAsync("update-status", async () =>
|
||||
{
|
||||
var customer = await db.Customers.FirstOrDefaultAsync(c => c.Id == ctx.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {ctx.CustomerId} not found.");
|
||||
var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct)
|
||||
?? throw new InvalidOperationException($"Instance {ctx.InstanceId} not found.");
|
||||
|
||||
customer.Status = CustomerStatus.Suspended;
|
||||
instance.HealthStatus = HealthStatus.Degraded;
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"Customer '{abbrev}' status → Suspended, instance health → Degraded.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: audit-log ───────────────────────────────────────────────
|
||||
await runner.RunAsync("audit-log", async () =>
|
||||
{
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/stripe-webhook",
|
||||
Action = "suspend",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"Subscription suspended. Services scaled to 0. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return "AuditLog entry written for suspension.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: broadcast ───────────────────────────────────────────────
|
||||
await runner.RunAsync("broadcast", async () =>
|
||||
{
|
||||
await hub.Clients.All.SendInstanceStatusChanged(
|
||||
ctx.CustomerId.ToString(), CustomerStatus.Suspended.ToString());
|
||||
return "Broadcast InstanceStatusChanged → Suspended.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation("SuspendPipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers (shared pattern from Phase1Pipeline)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(SettingsService settings)
|
||||
{
|
||||
var host = await settings.GetAsync("Ssh.SwarmHost")
|
||||
?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost).");
|
||||
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
|
||||
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
|
||||
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
|
||||
var password = await settings.GetAsync("Ssh.SwarmPassword");
|
||||
|
||||
if (!int.TryParse(portStr, out var port)) port = 22;
|
||||
|
||||
return new SshConnectionInfo(host, port, user, keyPath, password);
|
||||
}
|
||||
|
||||
private static SshClient CreateSshClient(SshConnectionInfo info)
|
||||
{
|
||||
var authMethods = new List<AuthenticationMethod>();
|
||||
|
||||
if (!string.IsNullOrEmpty(info.KeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
|
||||
|
||||
if (!string.IsNullOrEmpty(info.Password))
|
||||
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
|
||||
|
||||
if (authMethods.Count == 0)
|
||||
{
|
||||
var defaultKeyPath = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
|
||||
if (File.Exists(defaultKeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
|
||||
else
|
||||
throw new InvalidOperationException(
|
||||
$"No SSH authentication method available for {info.Host}:{info.Port}.");
|
||||
}
|
||||
|
||||
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
|
||||
return new SshClient(connInfo);
|
||||
}
|
||||
|
||||
private static string RunSshCommand(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
if (cmd.ExitStatus != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
|
||||
return cmd.Result;
|
||||
}
|
||||
|
||||
internal sealed record SshConnectionInfo(
|
||||
string Host, int Port, string Username, string? KeyPath, string? Password);
|
||||
}
|
||||
Reference in New Issue
Block a user