2026-02-18 10:43:27 -05:00
|
|
|
using System.Diagnostics;
|
|
|
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
|
using Microsoft.Extensions.Options;
|
|
|
|
|
using OTSSignsOrchestrator.Core.Configuration;
|
|
|
|
|
using OTSSignsOrchestrator.Core.Models.DTOs;
|
|
|
|
|
using OTSSignsOrchestrator.Core.Models.Entities;
|
|
|
|
|
using OTSSignsOrchestrator.Core.Services;
|
|
|
|
|
|
|
|
|
|
namespace OTSSignsOrchestrator.Desktop.Services;
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Docker CLI service that executes docker commands on a remote host over SSH.
|
|
|
|
|
/// Requires an SshHost to be set before use via SetHost().
|
|
|
|
|
/// </summary>
|
|
|
|
|
public class SshDockerCliService : IDockerCliService
|
|
|
|
|
{
|
|
|
|
|
private readonly SshConnectionService _ssh;
|
|
|
|
|
private readonly DockerOptions _options;
|
|
|
|
|
private readonly ILogger<SshDockerCliService> _logger;
|
|
|
|
|
private SshHost? _currentHost;
|
|
|
|
|
|
|
|
|
|
public SshDockerCliService(
|
|
|
|
|
SshConnectionService ssh,
|
|
|
|
|
IOptions<DockerOptions> options,
|
|
|
|
|
ILogger<SshDockerCliService> logger)
|
|
|
|
|
{
|
|
|
|
|
_ssh = ssh;
|
|
|
|
|
_options = options.Value;
|
|
|
|
|
_logger = logger;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Set the SSH host to use for Docker commands.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public void SetHost(SshHost host)
|
|
|
|
|
{
|
|
|
|
|
_currentHost = host;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public SshHost? CurrentHost => _currentHost;
|
|
|
|
|
|
|
|
|
|
public async Task<DeploymentResultDto> DeployStackAsync(string stackName, string composeYaml, bool resolveImage = false)
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
|
|
|
|
|
|
|
|
var args = "docker stack deploy --compose-file -";
|
|
|
|
|
if (resolveImage)
|
|
|
|
|
args += " --resolve-image changed";
|
|
|
|
|
args += $" {stackName}";
|
|
|
|
|
|
|
|
|
|
_logger.LogInformation("Deploying stack via SSH: {StackName} on {Host}", stackName, _currentHost!.Label);
|
|
|
|
|
|
|
|
|
|
var (exitCode, stdout, stderr) = await _ssh.RunCommandWithStdinAsync(_currentHost, args, composeYaml);
|
|
|
|
|
sw.Stop();
|
|
|
|
|
|
|
|
|
|
var result = new DeploymentResultDto
|
|
|
|
|
{
|
|
|
|
|
StackName = stackName,
|
|
|
|
|
Success = exitCode == 0,
|
|
|
|
|
ExitCode = exitCode,
|
|
|
|
|
Output = stdout,
|
|
|
|
|
ErrorMessage = stderr,
|
|
|
|
|
Message = exitCode == 0 ? "Success" : "Failed",
|
|
|
|
|
DurationMs = sw.ElapsedMilliseconds
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
if (result.Success)
|
|
|
|
|
_logger.LogInformation("Stack deployed via SSH: {StackName} | duration={DurationMs}ms", stackName, result.DurationMs);
|
|
|
|
|
else
|
|
|
|
|
_logger.LogError("Stack deploy failed via SSH: {StackName} | error={Error}", stackName, result.ErrorMessage);
|
|
|
|
|
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<DeploymentResultDto> RemoveStackAsync(string stackName)
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
|
|
|
|
|
|
|
|
_logger.LogInformation("Removing stack via SSH: {StackName} on {Host}", stackName, _currentHost!.Label);
|
|
|
|
|
|
|
|
|
|
var (exitCode, stdout, stderr) = await _ssh.RunCommandAsync(_currentHost, $"docker stack rm {stackName}");
|
|
|
|
|
sw.Stop();
|
|
|
|
|
|
|
|
|
|
var result = new DeploymentResultDto
|
|
|
|
|
{
|
|
|
|
|
StackName = stackName,
|
|
|
|
|
Success = exitCode == 0,
|
|
|
|
|
ExitCode = exitCode,
|
|
|
|
|
Output = stdout,
|
|
|
|
|
ErrorMessage = stderr,
|
|
|
|
|
Message = exitCode == 0 ? "Success" : "Failed",
|
|
|
|
|
DurationMs = sw.ElapsedMilliseconds
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
if (result.Success)
|
|
|
|
|
_logger.LogInformation("Stack removed via SSH: {StackName}", stackName);
|
|
|
|
|
else
|
|
|
|
|
_logger.LogError("Stack remove failed via SSH: {StackName} | error={Error}", stackName, result.ErrorMessage);
|
|
|
|
|
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<List<StackInfo>> ListStacksAsync()
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
|
|
|
|
|
var (exitCode, stdout, _) = await _ssh.RunCommandAsync(
|
|
|
|
|
_currentHost!, "docker stack ls --format '{{.Name}}\\t{{.Services}}'");
|
|
|
|
|
|
|
|
|
|
if (exitCode != 0 || string.IsNullOrWhiteSpace(stdout))
|
|
|
|
|
return new List<StackInfo>();
|
|
|
|
|
|
|
|
|
|
return stdout
|
|
|
|
|
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
|
|
|
|
|
.Select(line =>
|
|
|
|
|
{
|
|
|
|
|
var parts = line.Split('\t', 2);
|
|
|
|
|
return new StackInfo
|
|
|
|
|
{
|
|
|
|
|
Name = parts[0].Trim(),
|
|
|
|
|
ServiceCount = parts.Length > 1 && int.TryParse(parts[1].Trim(), out var c) ? c : 0
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
.ToList();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<List<ServiceInfo>> InspectStackServicesAsync(string stackName)
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
|
|
|
|
|
var (exitCode, stdout, _) = await _ssh.RunCommandAsync(
|
|
|
|
|
_currentHost!, $"docker stack services {stackName} --format '{{{{.Name}}}}\\t{{{{.Image}}}}\\t{{{{.Replicas}}}}'");
|
|
|
|
|
|
|
|
|
|
if (exitCode != 0 || string.IsNullOrWhiteSpace(stdout))
|
|
|
|
|
return new List<ServiceInfo>();
|
|
|
|
|
|
|
|
|
|
return stdout
|
|
|
|
|
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
|
|
|
|
|
.Select(line =>
|
|
|
|
|
{
|
|
|
|
|
var parts = line.Split('\t', 3);
|
|
|
|
|
return new ServiceInfo
|
|
|
|
|
{
|
|
|
|
|
Name = parts.Length > 0 ? parts[0].Trim() : "",
|
|
|
|
|
Image = parts.Length > 1 ? parts[1].Trim() : "",
|
|
|
|
|
Replicas = parts.Length > 2 ? parts[2].Trim() : ""
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
.ToList();
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-18 16:15:54 -05:00
|
|
|
public async Task<bool> EnsureDirectoryAsync(string path)
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
var (exitCode, _, stderr) = await _ssh.RunCommandAsync(_currentHost!, $"mkdir -p {path}");
|
|
|
|
|
if (exitCode != 0)
|
|
|
|
|
_logger.LogWarning("Failed to create directory {Path} on {Host}: {Error}", path, _currentHost!.Label, stderr);
|
|
|
|
|
else
|
|
|
|
|
_logger.LogInformation("Ensured directory exists on {Host}: {Path}", _currentHost!.Label, path);
|
|
|
|
|
return exitCode == 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<bool> EnsureSmbFoldersAsync(
|
|
|
|
|
string cifsServer,
|
|
|
|
|
string cifsShareName,
|
|
|
|
|
string cifsUsername,
|
|
|
|
|
string cifsPassword,
|
|
|
|
|
IEnumerable<string> folderNames,
|
|
|
|
|
string? cifsShareFolder = null)
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
var allSucceeded = true;
|
|
|
|
|
var subFolder = (cifsShareFolder ?? string.Empty).Trim('/');
|
|
|
|
|
|
|
|
|
|
// If a subfolder is specified, ensure it exists first
|
|
|
|
|
if (!string.IsNullOrEmpty(subFolder))
|
|
|
|
|
{
|
|
|
|
|
var mkdirCmd = $"smbclient //{cifsServer}/{cifsShareName} -U '{cifsUsername}%{cifsPassword}' -c 'mkdir {subFolder}' 2>&1";
|
|
|
|
|
var (_, mkdirOut, _) = await _ssh.RunCommandAsync(_currentHost!, mkdirCmd);
|
|
|
|
|
var mkdirOutput = mkdirOut ?? string.Empty;
|
|
|
|
|
|
|
|
|
|
var alreadyExists = mkdirOutput.Contains("NT_STATUS_OBJECT_NAME_COLLISION", StringComparison.OrdinalIgnoreCase)
|
|
|
|
|
|| mkdirOutput.Contains("already exists", StringComparison.OrdinalIgnoreCase);
|
|
|
|
|
var success = alreadyExists || !mkdirOutput.Contains("NT_STATUS_", StringComparison.OrdinalIgnoreCase);
|
|
|
|
|
|
|
|
|
|
if (success)
|
|
|
|
|
_logger.LogInformation("SMB subfolder ensured: //{Server}/{Share}/{Folder}", cifsServer, cifsShareName, subFolder);
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
_logger.LogWarning("Failed to create SMB subfolder //{Server}/{Share}/{Folder}: {Output}",
|
|
|
|
|
cifsServer, cifsShareName, subFolder, mkdirOutput.Trim());
|
|
|
|
|
allSucceeded = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Build the target path prefix for volume folders
|
|
|
|
|
var pathPrefix = string.IsNullOrEmpty(subFolder) ? string.Empty : $"{subFolder}/";
|
|
|
|
|
|
|
|
|
|
foreach (var folder in folderNames)
|
|
|
|
|
{
|
|
|
|
|
var targetFolder = $"{pathPrefix}{folder}";
|
|
|
|
|
// Run smbclient on the remote Docker host to create the folder on the share.
|
|
|
|
|
// NT_STATUS_OBJECT_NAME_COLLISION means it already exists — treat as success.
|
|
|
|
|
var cmd = $"smbclient //{cifsServer}/{cifsShareName} -U '{cifsUsername}%{cifsPassword}' -c 'mkdir {targetFolder}' 2>&1";
|
|
|
|
|
var (_, stdout, _) = await _ssh.RunCommandAsync(_currentHost!, cmd);
|
|
|
|
|
var output = stdout ?? string.Empty;
|
|
|
|
|
|
|
|
|
|
var exists = output.Contains("NT_STATUS_OBJECT_NAME_COLLISION", StringComparison.OrdinalIgnoreCase)
|
|
|
|
|
|| output.Contains("already exists", StringComparison.OrdinalIgnoreCase);
|
|
|
|
|
var ok = exists || !output.Contains("NT_STATUS_", StringComparison.OrdinalIgnoreCase);
|
|
|
|
|
|
|
|
|
|
if (ok)
|
|
|
|
|
_logger.LogInformation("SMB folder ensured: //{Server}/{Share}/{Folder}", cifsServer, cifsShareName, targetFolder);
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
_logger.LogWarning("Failed to create SMB folder //{Server}/{Share}/{Folder}: {Output}",
|
|
|
|
|
cifsServer, cifsShareName, targetFolder, output.Trim());
|
|
|
|
|
allSucceeded = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return allSucceeded;
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-18 10:43:27 -05:00
|
|
|
private void EnsureHost()
|
|
|
|
|
{
|
|
|
|
|
if (_currentHost == null)
|
|
|
|
|
throw new InvalidOperationException("No SSH host configured. Call SetHost() before using Docker commands.");
|
|
|
|
|
}
|
2026-02-18 16:15:54 -05:00
|
|
|
|
|
|
|
|
public async Task<bool> RemoveStackVolumesAsync(string stackName)
|
|
|
|
|
{
|
|
|
|
|
EnsureHost();
|
|
|
|
|
|
|
|
|
|
// ── 1. Remove the stack first so containers release the volumes ─────
|
|
|
|
|
_logger.LogInformation("Removing stack {StackName} before volume cleanup", stackName);
|
|
|
|
|
var (rmExit, _, rmErr) = await _ssh.RunCommandAsync(_currentHost!,
|
|
|
|
|
$"docker stack rm {stackName} 2>&1 || true");
|
|
|
|
|
if (rmExit != 0)
|
|
|
|
|
_logger.LogWarning("Stack rm returned non-zero for {StackName}: {Err}", stackName, rmErr);
|
|
|
|
|
|
|
|
|
|
// Give Swarm a moment to tear down containers on all nodes
|
|
|
|
|
await Task.Delay(5000);
|
|
|
|
|
|
|
|
|
|
// ── 2. Clean volumes on the local (manager) node ────────────────────
|
|
|
|
|
var localCmd = $"docker volume ls --filter \"name={stackName}_\" -q | xargs -r docker volume rm 2>&1 || true";
|
|
|
|
|
var (_, localOut, _) = await _ssh.RunCommandAsync(_currentHost!, localCmd);
|
|
|
|
|
if (!string.IsNullOrEmpty(localOut?.Trim()))
|
|
|
|
|
_logger.LogInformation("Volume cleanup (manager): {Output}", localOut!.Trim());
|
|
|
|
|
|
|
|
|
|
// ── 3. Clean volumes on ALL swarm nodes via a temporary global service ──
|
|
|
|
|
// This deploys a short-lived container on every node that mounts the Docker
|
|
|
|
|
// socket and removes matching volumes. This handles worker nodes that the
|
|
|
|
|
// orchestrator has no direct SSH access to.
|
|
|
|
|
var cleanupSvcName = $"vol-cleanup-{stackName}".Replace("_", "-");
|
|
|
|
|
|
|
|
|
|
// Remove leftover cleanup service from a previous run (if any)
|
|
|
|
|
await _ssh.RunCommandAsync(_currentHost!,
|
|
|
|
|
$"docker service rm {cleanupSvcName} 2>/dev/null || true");
|
|
|
|
|
|
|
|
|
|
var createCmd = string.Join(" ",
|
|
|
|
|
"docker service create",
|
|
|
|
|
"--detach",
|
|
|
|
|
"--mode global",
|
|
|
|
|
"--restart-condition none",
|
|
|
|
|
$"--name {cleanupSvcName}",
|
|
|
|
|
"--mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock",
|
|
|
|
|
"docker:cli",
|
|
|
|
|
"sh", "-c",
|
|
|
|
|
$"'docker volume ls -q --filter name={stackName}_ | xargs -r docker volume rm 2>&1; echo done'");
|
|
|
|
|
|
|
|
|
|
_logger.LogInformation("Deploying global volume-cleanup service on all swarm nodes for {StackName}", stackName);
|
|
|
|
|
var (svcExit, svcOut, svcErr) = await _ssh.RunCommandAsync(_currentHost!, createCmd);
|
|
|
|
|
|
|
|
|
|
if (svcExit != 0)
|
|
|
|
|
{
|
|
|
|
|
_logger.LogWarning("Global volume cleanup service creation failed: {Err}", svcErr);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
// Wait for the cleanup tasks to finish on all nodes
|
|
|
|
|
_logger.LogInformation("Waiting for volume cleanup tasks to complete on all nodes...");
|
|
|
|
|
await Task.Delay(10000);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Remove the cleanup service
|
|
|
|
|
await _ssh.RunCommandAsync(_currentHost!,
|
|
|
|
|
$"docker service rm {cleanupSvcName} 2>/dev/null || true");
|
|
|
|
|
|
|
|
|
|
_logger.LogInformation("Volume cleanup complete for stack {StackName}", stackName);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
2026-02-18 10:43:27 -05:00
|
|
|
}
|