feat: Implement provisioning pipelines for subscription management

- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes. - Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging. - Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR. - Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes. - Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots. - Introduce XiboFeatureManifests for hardcoded feature ACLs per role. - Add docker-compose.dev.yml for local development with PostgreSQL setup.
2026-03-18 10:27:26 -04:00
parent c2e03de8bb
commit c6d46098dd
77 changed files with 9412 additions and 29 deletions
--- a/OTSSignsOrchestrator.Server/Health/Checks/MySqlConnectHealthCheck.cs
+++ b/OTSSignsOrchestrator.Server/Health/Checks/MySqlConnectHealthCheck.cs
@@ -0,0 +1,106 @@
+using Renci.SshNet;
+using OTSSignsOrchestrator.Server.Data.Entities;
+
+namespace OTSSignsOrchestrator.Server.Health.Checks;
+
+/// <summary>
+/// Verifies connectivity to the instance's MySQL database by running a simple query
+/// via SSH against the Docker Swarm host.
+/// </summary>
+public sealed class MySqlConnectHealthCheck : IHealthCheck
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger<MySqlConnectHealthCheck> _logger;
+
+    public string CheckName => "MySqlConnect";
+    public bool AutoRemediate => false;
+
+    public MySqlConnectHealthCheck(
+        IServiceProvider services,
+        ILogger<MySqlConnectHealthCheck> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<HealthCheckResult> RunAsync(Instance instance, CancellationToken ct)
+    {
+        var dbName = instance.MysqlDatabase;
+        if (string.IsNullOrEmpty(dbName))
+            return new HealthCheckResult(HealthStatus.Critical, "No MySQL database configured");
+
+        try
+        {
+            var settings = _services.GetRequiredService<Core.Services.SettingsService>();
+            var sshInfo = await GetSwarmSshHostAsync(settings);
+            var mysqlHost = await settings.GetAsync(Core.Services.SettingsService.MySqlHost, "localhost");
+            var mysqlPort = await settings.GetAsync(Core.Services.SettingsService.MySqlPort, "3306");
+            var mysqlUser = await settings.GetAsync(Core.Services.SettingsService.MySqlAdminUser, "root");
+            var mysqlPass = await settings.GetAsync(Core.Services.SettingsService.MySqlAdminPassword, "");
+
+            using var sshClient = CreateSshClient(sshInfo);
+            sshClient.Connect();
+            try
+            {
+                // Simple connectivity test — SELECT 1 against the instance database
+                var cmd = $"mysql -h {mysqlHost} -P {mysqlPort} -u {mysqlUser} " +
+                          $"-p'{mysqlPass}' -e 'SELECT 1' {dbName} 2>&1";
+                var output = RunSshCommand(sshClient, cmd);
+
+                return new HealthCheckResult(HealthStatus.Healthy,
+                    $"MySQL connection to {dbName} successful");
+            }
+            finally
+            {
+                sshClient.Disconnect();
+            }
+        }
+        catch (Exception ex)
+        {
+            return new HealthCheckResult(HealthStatus.Critical,
+                $"MySQL connection failed for {dbName}: {ex.Message}");
+        }
+    }
+
+    private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(Core.Services.SettingsService settings)
+    {
+        var host = await settings.GetAsync("Ssh.SwarmHost")
+            ?? throw new InvalidOperationException("SSH Swarm host not configured.");
+        var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
+        var user = await settings.GetAsync("Ssh.SwarmUser", "root");
+        var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
+        var password = await settings.GetAsync("Ssh.SwarmPassword");
+        if (!int.TryParse(portStr, out var port)) port = 22;
+        return new SshConnectionInfo(host, port, user, keyPath, password);
+    }
+
+    private static SshClient CreateSshClient(SshConnectionInfo info)
+    {
+        var authMethods = new List<AuthenticationMethod>();
+        if (!string.IsNullOrEmpty(info.KeyPath))
+            authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
+        if (!string.IsNullOrEmpty(info.Password))
+            authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
+        if (authMethods.Count == 0)
+        {
+            var defaultKeyPath = Path.Combine(
+                Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
+            if (File.Exists(defaultKeyPath))
+                authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
+            else
+                throw new InvalidOperationException($"No SSH auth method for {info.Host}:{info.Port}.");
+        }
+        var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
+        return new SshClient(connInfo);
+    }
+
+    private static string RunSshCommand(SshClient client, string command)
+    {
+        using var cmd = client.RunCommand(command);
+        if (cmd.ExitStatus != 0)
+            throw new InvalidOperationException($"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
+        return cmd.Result;
+    }
+
+    internal sealed record SshConnectionInfo(string Host, int Port, string Username, string? KeyPath, string? Password);
+}