feat: Implement provisioning pipelines for subscription management

- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes. - Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging. - Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR. - Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes. - Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots. - Introduce XiboFeatureManifests for hardcoded feature ACLs per role. - Add docker-compose.dev.yml for local development with PostgreSQL setup.
2026-03-18 10:27:26 -04:00
parent c2e03de8bb
commit c6d46098dd
77 changed files with 9412 additions and 29 deletions
--- a/OTSSignsOrchestrator.Server/Health/Checks/NfsAccessHealthCheck.cs
+++ b/OTSSignsOrchestrator.Server/Health/Checks/NfsAccessHealthCheck.cs
@@ -0,0 +1,121 @@
+using Renci.SshNet;
+using OTSSignsOrchestrator.Server.Data.Entities;
+
+namespace OTSSignsOrchestrator.Server.Health.Checks;
+
+/// <summary>
+/// Verifies NFS paths for the instance are accessible by running <c>ls</c> via SSH.
+/// </summary>
+public sealed class NfsAccessHealthCheck : IHealthCheck
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger<NfsAccessHealthCheck> _logger;
+
+    public string CheckName => "NfsAccess";
+    public bool AutoRemediate => false;
+
+    public NfsAccessHealthCheck(
+        IServiceProvider services,
+        ILogger<NfsAccessHealthCheck> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<HealthCheckResult> RunAsync(Instance instance, CancellationToken ct)
+    {
+        var nfsPath = instance.NfsPath;
+        if (string.IsNullOrEmpty(nfsPath))
+            return new HealthCheckResult(HealthStatus.Critical, "No NFS path configured");
+
+        try
+        {
+            var settings = _services.GetRequiredService<Core.Services.SettingsService>();
+            var sshInfo = await GetSwarmSshHostAsync(settings);
+            var nfsServer = await settings.GetAsync(Core.Services.SettingsService.NfsServer);
+            var nfsExport = await settings.GetAsync(Core.Services.SettingsService.NfsExport);
+
+            if (string.IsNullOrEmpty(nfsServer) || string.IsNullOrEmpty(nfsExport))
+                return new HealthCheckResult(HealthStatus.Critical, "NFS server/export not configured");
+
+            using var sshClient = CreateSshClient(sshInfo);
+            sshClient.Connect();
+            try
+            {
+                // Mount temporarily and check the path is listable
+                var mountPoint = $"/tmp/healthcheck-nfs-{Guid.NewGuid():N}";
+                RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}");
+                try
+                {
+                    RunSshCommand(sshClient, $"sudo mount -t nfs4 {nfsServer}:{nfsExport} {mountPoint}");
+                    var output = RunSshCommand(sshClient, $"ls {mountPoint}/{nfsPath} 2>&1");
+
+                    return new HealthCheckResult(HealthStatus.Healthy,
+                        $"NFS path accessible: {nfsPath}");
+                }
+                finally
+                {
+                    RunSshCommandAllowFailure(sshClient, $"sudo umount {mountPoint}");
+                    RunSshCommandAllowFailure(sshClient, $"sudo rmdir {mountPoint}");
+                }
+            }
+            finally
+            {
+                sshClient.Disconnect();
+            }
+        }
+        catch (Exception ex)
+        {
+            return new HealthCheckResult(HealthStatus.Critical,
+                $"NFS access check failed for {nfsPath}: {ex.Message}");
+        }
+    }
+
+    private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(Core.Services.SettingsService settings)
+    {
+        var host = await settings.GetAsync("Ssh.SwarmHost")
+            ?? throw new InvalidOperationException("SSH Swarm host not configured.");
+        var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
+        var user = await settings.GetAsync("Ssh.SwarmUser", "root");
+        var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
+        var password = await settings.GetAsync("Ssh.SwarmPassword");
+        if (!int.TryParse(portStr, out var port)) port = 22;
+        return new SshConnectionInfo(host, port, user, keyPath, password);
+    }
+
+    private static SshClient CreateSshClient(SshConnectionInfo info)
+    {
+        var authMethods = new List<AuthenticationMethod>();
+        if (!string.IsNullOrEmpty(info.KeyPath))
+            authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
+        if (!string.IsNullOrEmpty(info.Password))
+            authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
+        if (authMethods.Count == 0)
+        {
+            var defaultKeyPath = Path.Combine(
+                Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
+            if (File.Exists(defaultKeyPath))
+                authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
+            else
+                throw new InvalidOperationException($"No SSH auth method for {info.Host}:{info.Port}.");
+        }
+        var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
+        return new SshClient(connInfo);
+    }
+
+    private static string RunSshCommand(SshClient client, string command)
+    {
+        using var cmd = client.RunCommand(command);
+        if (cmd.ExitStatus != 0)
+            throw new InvalidOperationException($"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
+        return cmd.Result;
+    }
+
+    private static void RunSshCommandAllowFailure(SshClient client, string command)
+    {
+        using var cmd = client.RunCommand(command);
+        // Intentionally ignore exit code — cleanup operations
+    }
+
+    internal sealed record SshConnectionInfo(string Host, int Port, string Username, string? KeyPath, string? Password);
+}