using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Renci.SshNet; using OTSSignsOrchestrator.Core.Services; using OTSSignsOrchestrator.Server.Data; using OTSSignsOrchestrator.Server.Data.Entities; using OTSSignsOrchestrator.Server.Hubs; namespace OTSSignsOrchestrator.Server.Workers; /// /// Subscription reactivation pipeline — scales up Docker services, verifies health, resets /// payment failure counters. Handles JobType = "reactivate". /// /// Steps: /// 1. scale-up — SSH docker service scale web=1, xmr=1 /// 2. health-verify — Poll GET /about every 10s up to 3 minutes /// 3. update-status — Customer.Status = Active, reset FailedPaymentCount + FirstPaymentFailedAt /// 4. audit-log — Append-only AuditLog entry /// 5. broadcast — InstanceStatusChanged via FleetHub /// public sealed class ReactivatePipeline : IProvisioningPipeline { public string HandlesJobType => "reactivate"; private const int TotalSteps = 5; private readonly IServiceProvider _services; private readonly ILogger _logger; public ReactivatePipeline( IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } public async Task ExecuteAsync(Job job, CancellationToken ct) { await using var scope = _services.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); var hub = scope.ServiceProvider.GetRequiredService>(); var settings = scope.ServiceProvider.GetRequiredService(); var ctx = await BuildContextAsync(job, db, ct); var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps); var abbrev = ctx.Abbreviation; // ── Step 1: scale-up ──────────────────────────────────────────────── await runner.RunAsync("scale-up", async () => { var sshHost = await GetSwarmSshHostAsync(settings); using var sshClient = CreateSshClient(sshHost); sshClient.Connect(); try { var cmd = $"docker service scale xibo-{abbrev}_web=1 xibo-{abbrev}_xmr=1"; var result = RunSshCommand(sshClient, cmd); return $"Scaled up services for xibo-{abbrev}. Output: {result}"; } finally { sshClient.Disconnect(); } }, ct); // ── Step 2: health-verify ─────────────────────────────────────────── await runner.RunAsync("health-verify", async () => { var timeout = TimeSpan.FromMinutes(3); var interval = TimeSpan.FromSeconds(10); var deadline = DateTimeOffset.UtcNow.Add(timeout); using var httpClient = new HttpClient { BaseAddress = new Uri(ctx.InstanceUrl.TrimEnd('/')) }; while (DateTimeOffset.UtcNow < deadline) { ct.ThrowIfCancellationRequested(); try { var response = await httpClient.GetAsync("/api/about", ct); if (response.IsSuccessStatusCode) return $"Xibo CMS at {ctx.InstanceUrl} is healthy (status {(int)response.StatusCode})."; } catch (HttpRequestException) { // Not ready yet — keep polling } await Task.Delay(interval, ct); } throw new TimeoutException( $"Xibo CMS at {ctx.InstanceUrl} did not return 200 OK from /about within {timeout.TotalMinutes} minutes."); }, ct); // ── Step 3: update-status ─────────────────────────────────────────── await runner.RunAsync("update-status", async () => { var customer = await db.Customers.FirstOrDefaultAsync(c => c.Id == ctx.CustomerId, ct) ?? throw new InvalidOperationException($"Customer {ctx.CustomerId} not found."); var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct) ?? throw new InvalidOperationException($"Instance {ctx.InstanceId} not found."); customer.Status = CustomerStatus.Active; customer.FailedPaymentCount = 0; customer.FirstPaymentFailedAt = null; instance.HealthStatus = HealthStatus.Healthy; instance.LastHealthCheck = DateTime.UtcNow; await db.SaveChangesAsync(ct); return $"Customer '{abbrev}' status → Active, FailedPaymentCount → 0, instance health → Healthy."; }, ct); // ── Step 4: audit-log ─────────────────────────────────────────────── await runner.RunAsync("audit-log", async () => { db.AuditLogs.Add(new AuditLog { Id = Guid.NewGuid(), InstanceId = ctx.InstanceId, Actor = "system/stripe-webhook", Action = "reactivate", Target = $"xibo-{abbrev}", Outcome = "success", Detail = $"Subscription reactivated. Services scaled to 1. Health verified. Job {job.Id}.", OccurredAt = DateTime.UtcNow, }); await db.SaveChangesAsync(ct); return "AuditLog entry written for reactivation."; }, ct); // ── Step 5: broadcast ─────────────────────────────────────────────── await runner.RunAsync("broadcast", async () => { await hub.Clients.All.SendInstanceStatusChanged( ctx.CustomerId.ToString(), CustomerStatus.Active.ToString()); return "Broadcast InstanceStatusChanged → Active."; }, ct); _logger.LogInformation("ReactivatePipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev); } // ───────────────────────────────────────────────────────────────────────── // Helpers (shared pattern from Phase1Pipeline) // ───────────────────────────────────────────────────────────────────────── private static async Task BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct) { var customer = await db.Customers .Include(c => c.Instances) .FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct) ?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}."); var instance = customer.Instances.FirstOrDefault() ?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}."); var abbrev = customer.Abbreviation.ToLowerInvariant(); return new PipelineContext { JobId = job.Id, CustomerId = customer.Id, InstanceId = instance.Id, Abbreviation = abbrev, CompanyName = customer.CompanyName, AdminEmail = customer.AdminEmail, AdminFirstName = customer.AdminFirstName, InstanceUrl = instance.XiboUrl, DockerStackName = instance.DockerStackName, ParametersJson = job.Parameters, }; } private static async Task GetSwarmSshHostAsync(SettingsService settings) { var host = await settings.GetAsync("Ssh.SwarmHost") ?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost)."); var portStr = await settings.GetAsync("Ssh.SwarmPort", "22"); var user = await settings.GetAsync("Ssh.SwarmUser", "root"); var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath"); var password = await settings.GetAsync("Ssh.SwarmPassword"); if (!int.TryParse(portStr, out var port)) port = 22; return new SshConnectionInfo(host, port, user, keyPath, password); } private static SshClient CreateSshClient(SshConnectionInfo info) { var authMethods = new List(); if (!string.IsNullOrEmpty(info.KeyPath)) authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath))); if (!string.IsNullOrEmpty(info.Password)) authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password)); if (authMethods.Count == 0) { var defaultKeyPath = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa"); if (File.Exists(defaultKeyPath)) authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath))); else throw new InvalidOperationException( $"No SSH authentication method available for {info.Host}:{info.Port}."); } var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray()); return new SshClient(connInfo); } private static string RunSshCommand(SshClient client, string command) { using var cmd = client.RunCommand(command); if (cmd.ExitStatus != 0) throw new InvalidOperationException( $"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}"); return cmd.Result; } internal sealed record SshConnectionInfo( string Host, int Port, string Username, string? KeyPath, string? Password); }