- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes. - Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging. - Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR. - Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes. - Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots. - Introduce XiboFeatureManifests for hardcoded feature ACLs per role. - Add docker-compose.dev.yml for local development with PostgreSQL setup.
122 lines
5.1 KiB
C#
122 lines
5.1 KiB
C#
using Renci.SshNet;
|
|
using OTSSignsOrchestrator.Server.Data.Entities;
|
|
|
|
namespace OTSSignsOrchestrator.Server.Health.Checks;
|
|
|
|
/// <summary>
|
|
/// Verifies NFS paths for the instance are accessible by running <c>ls</c> via SSH.
|
|
/// </summary>
|
|
public sealed class NfsAccessHealthCheck : IHealthCheck
|
|
{
|
|
private readonly IServiceProvider _services;
|
|
private readonly ILogger<NfsAccessHealthCheck> _logger;
|
|
|
|
public string CheckName => "NfsAccess";
|
|
public bool AutoRemediate => false;
|
|
|
|
public NfsAccessHealthCheck(
|
|
IServiceProvider services,
|
|
ILogger<NfsAccessHealthCheck> logger)
|
|
{
|
|
_services = services;
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<HealthCheckResult> RunAsync(Instance instance, CancellationToken ct)
|
|
{
|
|
var nfsPath = instance.NfsPath;
|
|
if (string.IsNullOrEmpty(nfsPath))
|
|
return new HealthCheckResult(HealthStatus.Critical, "No NFS path configured");
|
|
|
|
try
|
|
{
|
|
var settings = _services.GetRequiredService<Core.Services.SettingsService>();
|
|
var sshInfo = await GetSwarmSshHostAsync(settings);
|
|
var nfsServer = await settings.GetAsync(Core.Services.SettingsService.NfsServer);
|
|
var nfsExport = await settings.GetAsync(Core.Services.SettingsService.NfsExport);
|
|
|
|
if (string.IsNullOrEmpty(nfsServer) || string.IsNullOrEmpty(nfsExport))
|
|
return new HealthCheckResult(HealthStatus.Critical, "NFS server/export not configured");
|
|
|
|
using var sshClient = CreateSshClient(sshInfo);
|
|
sshClient.Connect();
|
|
try
|
|
{
|
|
// Mount temporarily and check the path is listable
|
|
var mountPoint = $"/tmp/healthcheck-nfs-{Guid.NewGuid():N}";
|
|
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}");
|
|
try
|
|
{
|
|
RunSshCommand(sshClient, $"sudo mount -t nfs4 {nfsServer}:{nfsExport} {mountPoint}");
|
|
var output = RunSshCommand(sshClient, $"ls {mountPoint}/{nfsPath} 2>&1");
|
|
|
|
return new HealthCheckResult(HealthStatus.Healthy,
|
|
$"NFS path accessible: {nfsPath}");
|
|
}
|
|
finally
|
|
{
|
|
RunSshCommandAllowFailure(sshClient, $"sudo umount {mountPoint}");
|
|
RunSshCommandAllowFailure(sshClient, $"sudo rmdir {mountPoint}");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
sshClient.Disconnect();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return new HealthCheckResult(HealthStatus.Critical,
|
|
$"NFS access check failed for {nfsPath}: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(Core.Services.SettingsService settings)
|
|
{
|
|
var host = await settings.GetAsync("Ssh.SwarmHost")
|
|
?? throw new InvalidOperationException("SSH Swarm host not configured.");
|
|
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
|
|
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
|
|
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
|
|
var password = await settings.GetAsync("Ssh.SwarmPassword");
|
|
if (!int.TryParse(portStr, out var port)) port = 22;
|
|
return new SshConnectionInfo(host, port, user, keyPath, password);
|
|
}
|
|
|
|
private static SshClient CreateSshClient(SshConnectionInfo info)
|
|
{
|
|
var authMethods = new List<AuthenticationMethod>();
|
|
if (!string.IsNullOrEmpty(info.KeyPath))
|
|
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
|
|
if (!string.IsNullOrEmpty(info.Password))
|
|
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
|
|
if (authMethods.Count == 0)
|
|
{
|
|
var defaultKeyPath = Path.Combine(
|
|
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
|
|
if (File.Exists(defaultKeyPath))
|
|
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
|
|
else
|
|
throw new InvalidOperationException($"No SSH auth method for {info.Host}:{info.Port}.");
|
|
}
|
|
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
|
|
return new SshClient(connInfo);
|
|
}
|
|
|
|
private static string RunSshCommand(SshClient client, string command)
|
|
{
|
|
using var cmd = client.RunCommand(command);
|
|
if (cmd.ExitStatus != 0)
|
|
throw new InvalidOperationException($"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
|
|
return cmd.Result;
|
|
}
|
|
|
|
private static void RunSshCommandAllowFailure(SshClient client, string command)
|
|
{
|
|
using var cmd = client.RunCommand(command);
|
|
// Intentionally ignore exit code — cleanup operations
|
|
}
|
|
|
|
internal sealed record SshConnectionInfo(string Host, int Port, string Username, string? KeyPath, string? Password);
|
|
}
|