5a0199a598
Some checks failed
Build and Publish Docker Image / build-and-push (push) Has been cancelled
745 lines
41 KiB
C#
745 lines
41 KiB
C#
using System.Diagnostics;
|
|
using System.Security.Cryptography;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using MySqlConnector;
|
|
using OTSSignsOrchestrator.Core.Configuration;
|
|
using OTSSignsOrchestrator.Core.Data;
|
|
using OTSSignsOrchestrator.Core.Models.DTOs;
|
|
using OTSSignsOrchestrator.Core.Models.Entities;
|
|
using static OTSSignsOrchestrator.Core.Configuration.AppConstants;
|
|
|
|
namespace OTSSignsOrchestrator.Core.Services;
|
|
|
|
/// <summary>
|
|
/// Orchestrator service for CMS instance lifecycle (Create, Update, Delete).
|
|
/// All instance data is live — nothing is persisted locally beyond the audit log.
|
|
/// New-instance flow:
|
|
/// 1. Clone template repo to local cache
|
|
/// 2. Generate MySQL password → create Docker Swarm secret (never persisted locally)
|
|
/// 3. Create MySQL database + user on external MySQL server via SSH
|
|
/// 4. Render combined compose YAML (no MySQL container, NFS volumes, Newt service)
|
|
/// 5. Deploy stack via SSH
|
|
/// </summary>
|
|
public class InstanceService
|
|
{
|
|
private readonly XiboContext _db;
|
|
private readonly GitTemplateService _git;
|
|
private readonly ComposeRenderService _compose;
|
|
private readonly ComposeValidationService _validation;
|
|
private readonly IDockerCliService _docker;
|
|
private readonly IDockerSecretsService _secrets;
|
|
private readonly XiboApiService _xibo;
|
|
private readonly SettingsService _settings;
|
|
private readonly DockerOptions _dockerOptions;
|
|
private readonly ILogger<InstanceService> _logger;
|
|
|
|
public InstanceService(
|
|
XiboContext db,
|
|
GitTemplateService git,
|
|
ComposeRenderService compose,
|
|
ComposeValidationService validation,
|
|
IDockerCliService docker,
|
|
IDockerSecretsService secrets,
|
|
XiboApiService xibo,
|
|
SettingsService settings,
|
|
IOptions<DockerOptions> dockerOptions,
|
|
ILogger<InstanceService> logger)
|
|
{
|
|
_db = db;
|
|
_git = git;
|
|
_compose = compose;
|
|
_validation = validation;
|
|
_docker = docker;
|
|
_secrets = secrets;
|
|
_xibo = xibo;
|
|
_settings = settings;
|
|
_dockerOptions = dockerOptions.Value;
|
|
_logger = logger;
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Helper: derive the 3-letter abbreviation from a stack name like "acm-cms-stack"
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
private static string ExtractAbbrev(string stackName)
|
|
=> stackName.EndsWith("-cms-stack")
|
|
? stackName[..^"-cms-stack".Length]
|
|
: stackName.Split('-')[0];
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Create
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Creates a new CMS instance:
|
|
/// 1. Clone repo 2. Generate secrets 3. Create MySQL DB/user 4. Render compose 5. Deploy
|
|
/// </summary>
|
|
public async Task<DeploymentResultDto> CreateInstanceAsync(CreateInstanceDto dto, string? userId = null)
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
var abbrev = dto.CustomerAbbrev.Trim().ToLowerInvariant();
|
|
var stackName = $"{abbrev}-cms-stack";
|
|
var opLog = StartOperation(OperationType.Create, userId, stackName);
|
|
|
|
try
|
|
{
|
|
_logger.LogInformation("Creating instance: abbrev={Abbrev}, customer={Customer}", abbrev, dto.CustomerName);
|
|
|
|
// ── 1. Clone / refresh template repo ────────────────────────────
|
|
var repoUrl = await _settings.GetAsync(SettingsService.GitRepoUrl);
|
|
var repoPat = await _settings.GetAsync(SettingsService.GitRepoPat);
|
|
|
|
if (string.IsNullOrWhiteSpace(repoUrl))
|
|
throw new InvalidOperationException("Git template repository URL is not configured. Set it in Settings → Git Repo URL.");
|
|
|
|
_logger.LogInformation("Fetching template repo: {RepoUrl}", repoUrl);
|
|
var templateConfig = await _git.FetchAsync(repoUrl, repoPat);
|
|
|
|
// ── 1b. Remove any stale stack that might hold references to old secrets ─
|
|
_logger.LogInformation("Removing stale stack (if any): {StackName}", stackName);
|
|
await _docker.RemoveStackAsync(stackName);
|
|
await Task.Delay(2000);
|
|
|
|
// ── 2. Generate MySQL credentials ──────────────────────────────
|
|
var mysqlPassword = GenerateRandomPassword(32);
|
|
var mySqlUserName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);
|
|
|
|
var userSecretName = CustomerMysqlUserSecretName(abbrev);
|
|
|
|
var (userOk, _) = await _secrets.EnsureSecretAsync(userSecretName, mySqlUserName, rotate: true);
|
|
if (!userOk)
|
|
throw new InvalidOperationException($"Failed to create/rotate Docker secret '{userSecretName}'.");
|
|
_logger.LogInformation("Docker secret created/rotated: {SecretName}", userSecretName);
|
|
|
|
// Global secrets: MySQL host + port (idempotent, shared across instances)
|
|
var mySqlHostValue = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
|
var mySqlPortValue = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
|
|
await _secrets.EnsureSecretAsync(GlobalMysqlHostSecretName, mySqlHostValue);
|
|
await _secrets.EnsureSecretAsync(GlobalMysqlPortSecretName, mySqlPortValue);
|
|
_logger.LogInformation("Global MySQL secrets ensured: {Host}, {Port}", GlobalMysqlHostSecretName, GlobalMysqlPortSecretName);
|
|
|
|
// ── 2b. Create MySQL database + user ─────────────────────────────
|
|
_logger.LogInformation("Creating MySQL database and user for instance {Abbrev}", abbrev);
|
|
var (mysqlOk, mysqlMsg) = await CreateMySqlDatabaseAsync(abbrev, mysqlPassword);
|
|
if (!mysqlOk)
|
|
throw new InvalidOperationException($"MySQL database/user setup failed: {mysqlMsg}");
|
|
_logger.LogInformation("MySQL setup complete: {Message}", mysqlMsg);
|
|
|
|
// ── 2c. Persist password (encrypted) for future redeploys ────────
|
|
await _settings.SetAsync(
|
|
SettingsService.InstanceMySqlPassword(abbrev), mysqlPassword,
|
|
SettingsService.CatInstance, isSensitive: true);
|
|
await _db.SaveChangesAsync();
|
|
_logger.LogInformation("MySQL password stored in settings for instance {Abbrev}", abbrev);
|
|
|
|
// ── 3. Read settings ────────────────────────────────────────────
|
|
var mySqlHost = mySqlHostValue;
|
|
var mySqlPort = mySqlPortValue;
|
|
var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
|
|
var mySqlUser = mySqlUserName;
|
|
|
|
var cmsServerName = (await _settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "{abbrev}.ots-signs.com")).Replace("{abbrev}", abbrev);
|
|
var themePath = (await _settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme")).Replace("{abbrev}", abbrev);
|
|
|
|
var smtpServer = await _settings.GetAsync(SettingsService.SmtpServer, string.Empty);
|
|
var smtpUsername = await _settings.GetAsync(SettingsService.SmtpUsername, string.Empty);
|
|
var smtpPassword = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty);
|
|
var smtpUseTls = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES");
|
|
var smtpUseStartTls = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES");
|
|
var smtpRewriteDomain = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty);
|
|
var smtpHostname = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty);
|
|
var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO");
|
|
|
|
var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net");
|
|
|
|
var nfsServer = dto.NfsServer ?? await _settings.GetAsync(SettingsService.NfsServer);
|
|
var nfsExport = dto.NfsExport ?? await _settings.GetAsync(SettingsService.NfsExport);
|
|
var nfsExportFolder = dto.NfsExportFolder ?? await _settings.GetAsync(SettingsService.NfsExportFolder);
|
|
var nfsOptions = dto.NfsExtraOptions ?? await _settings.GetAsync(SettingsService.NfsOptions, string.Empty);
|
|
|
|
var cmsImage = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3");
|
|
var newtImage = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt");
|
|
var memcachedImage = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine");
|
|
var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart");
|
|
|
|
var phpPostMaxSize = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G");
|
|
var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G");
|
|
var phpMaxExecutionTime = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600");
|
|
|
|
// ── 4. Render compose YAML from template ────────────────────────
|
|
var renderCtx = new RenderContext
|
|
{
|
|
CustomerName = dto.CustomerName,
|
|
CustomerAbbrev = abbrev,
|
|
StackName = stackName,
|
|
CmsServerName = cmsServerName,
|
|
HostHttpPort = 80,
|
|
CmsImage = cmsImage,
|
|
MemcachedImage = memcachedImage,
|
|
QuickChartImage = quickChartImage,
|
|
NewtImage = newtImage,
|
|
ThemeHostPath = themePath,
|
|
MySqlHost = mySqlHost,
|
|
MySqlPort = mySqlPort,
|
|
MySqlDatabase = mySqlDbName,
|
|
MySqlUser = mySqlUser,
|
|
MySqlPassword = mysqlPassword,
|
|
SmtpServer = smtpServer,
|
|
SmtpUsername = smtpUsername,
|
|
SmtpPassword = smtpPassword,
|
|
SmtpUseTls = smtpUseTls,
|
|
SmtpUseStartTls = smtpUseStartTls,
|
|
SmtpRewriteDomain = smtpRewriteDomain,
|
|
SmtpHostname = smtpHostname,
|
|
SmtpFromLineOverride = smtpFromLineOverride,
|
|
PhpPostMaxSize = phpPostMaxSize,
|
|
PhpUploadMaxFilesize = phpUploadMaxFilesize,
|
|
PhpMaxExecutionTime = phpMaxExecutionTime,
|
|
PangolinEndpoint = pangolinEndpoint,
|
|
NewtId = dto.NewtId,
|
|
NewtSecret = dto.NewtSecret,
|
|
NfsServer = nfsServer,
|
|
NfsExport = nfsExport,
|
|
NfsExportFolder = nfsExportFolder,
|
|
NfsExtraOptions = nfsOptions,
|
|
};
|
|
|
|
_logger.LogInformation("NFS render values: server={NfsServer}, export={NfsExport}, folder={NfsExportFolder}",
|
|
nfsServer, nfsExport, nfsExportFolder);
|
|
|
|
var composeYaml = _compose.Render(templateConfig.Yaml, renderCtx);
|
|
|
|
if (_dockerOptions.ValidateBeforeDeploy)
|
|
{
|
|
var validationResult = _validation.Validate(composeYaml, abbrev);
|
|
if (!validationResult.IsValid)
|
|
throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}");
|
|
}
|
|
|
|
// ── 5. Ensure bind-mount directories exist on the remote host ───
|
|
if (!string.IsNullOrWhiteSpace(themePath))
|
|
await _docker.EnsureDirectoryAsync(themePath);
|
|
|
|
// ── 5b. Ensure NFS export folders exist ─────────────────────────
|
|
if (!string.IsNullOrWhiteSpace(nfsServer) && !string.IsNullOrWhiteSpace(nfsExport))
|
|
{
|
|
var nfsFolders = ComposeRenderService.ExtractNfsDeviceFolders(composeYaml, nfsExport, nfsExportFolder);
|
|
_logger.LogInformation("Ensuring NFS export folders exist on {Server}:{Export} — {Folders}",
|
|
nfsServer, nfsExport, string.Join(", ", nfsFolders));
|
|
var (nfsFoldersOk, nfsError) = await _docker.EnsureNfsFoldersWithErrorAsync(nfsServer, nfsExport, nfsFolders, nfsExportFolder);
|
|
if (!nfsFoldersOk)
|
|
throw new InvalidOperationException(
|
|
$"Failed to create NFS volume directories on {nfsServer}:{nfsExport}: {nfsError}\n"
|
|
+ "Common causes: (1) SSH user needs passwordless sudo (NOPASSWD) for mount/umount/mkdir, "
|
|
+ "(2) NFS export has root_squash enabled — set 'No mapping' / no_root_squash on the NFS server.");
|
|
}
|
|
|
|
// ── 6. Remove stale NFS volumes ─────────────────────────────────
|
|
_logger.LogInformation("Removing stale NFS volumes for stack {StackName}", stackName);
|
|
await _docker.RemoveStackVolumesAsync(stackName);
|
|
|
|
// ── 7. Deploy stack ─────────────────────────────────────────────
|
|
var deployResult = await _docker.DeployStackAsync(stackName, composeYaml);
|
|
if (!deployResult.Success)
|
|
throw new InvalidOperationException($"Stack deployment failed: {deployResult.ErrorMessage}");
|
|
|
|
mysqlPassword = string.Empty;
|
|
|
|
sw.Stop();
|
|
opLog.Status = OperationStatus.Success;
|
|
opLog.Message = $"Instance deployed: {stackName}";
|
|
opLog.DurationMs = sw.ElapsedMilliseconds;
|
|
_db.OperationLogs.Add(opLog);
|
|
await _db.SaveChangesAsync();
|
|
|
|
_logger.LogInformation("Instance created: {StackName} | duration={DurationMs}ms", stackName, sw.ElapsedMilliseconds);
|
|
|
|
deployResult.ServiceCount = 4;
|
|
deployResult.Message = "Instance deployed successfully.";
|
|
return deployResult;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
sw.Stop();
|
|
opLog.Status = OperationStatus.Failure;
|
|
opLog.Message = $"Create failed: {ex.Message}";
|
|
opLog.DurationMs = sw.ElapsedMilliseconds;
|
|
_db.OperationLogs.Add(opLog);
|
|
await _db.SaveChangesAsync();
|
|
_logger.LogError(ex, "Instance create failed: {StackName}", stackName);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Update / redeploy
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Redeploys an existing stack. All render parameters come from <paramref name="dto"/>
|
|
/// (populated from live service inspection or user input), falling back to global settings
|
|
/// where values are null.
|
|
/// </summary>
|
|
public async Task<DeploymentResultDto> UpdateInstanceAsync(string stackName, UpdateInstanceDto dto, string? userId = null)
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
var opLog = StartOperation(OperationType.Update, userId, stackName);
|
|
|
|
try
|
|
{
|
|
var abbrev = dto.CustomerAbbrev?.Trim().ToLowerInvariant() ?? ExtractAbbrev(stackName);
|
|
var customerName = dto.CustomerName ?? abbrev;
|
|
|
|
_logger.LogInformation("Updating instance: {StackName} (abbrev={Abbrev})", stackName, abbrev);
|
|
|
|
var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
|
var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
|
|
var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
|
|
var mySqlUser = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);
|
|
|
|
// Retrieve the stored MySQL password (encrypted in settings since creation)
|
|
var mySqlPassword = await _settings.GetAsync(SettingsService.InstanceMySqlPassword(abbrev));
|
|
if (string.IsNullOrEmpty(mySqlPassword))
|
|
throw new InvalidOperationException(
|
|
$"No stored MySQL password found for instance '{abbrev}'. " +
|
|
"The instance may have been created before password persistence was added. " +
|
|
"Use 'Rotate Password' to generate and store a new one.");
|
|
|
|
// Ensure MySQL Docker secrets exist (idempotent)
|
|
var userSecretName = CustomerMysqlUserSecretName(abbrev);
|
|
await _secrets.EnsureSecretAsync(userSecretName, mySqlUser);
|
|
await _secrets.EnsureSecretAsync(GlobalMysqlHostSecretName, mySqlHost);
|
|
await _secrets.EnsureSecretAsync(GlobalMysqlPortSecretName, mySqlPort);
|
|
|
|
var smtpServer = dto.SmtpServer ?? await _settings.GetAsync(SettingsService.SmtpServer, string.Empty);
|
|
var smtpUsername = dto.SmtpUsername ?? await _settings.GetAsync(SettingsService.SmtpUsername, string.Empty);
|
|
var smtpPassword = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty);
|
|
var smtpUseTls = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES");
|
|
var smtpUseStartTls = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES");
|
|
var smtpRewriteDomain = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty);
|
|
var smtpHostname = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty);
|
|
var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO");
|
|
|
|
var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net");
|
|
|
|
var cmsServerName = dto.CmsServerName
|
|
?? (await _settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "{abbrev}.ots-signs.com")).Replace("{abbrev}", abbrev);
|
|
var hostHttpPort = dto.HostHttpPort ?? 80;
|
|
var themePath = dto.ThemeHostPath
|
|
?? (await _settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme")).Replace("{abbrev}", abbrev);
|
|
|
|
var nfsServer = dto.NfsServer ?? await _settings.GetAsync(SettingsService.NfsServer);
|
|
var nfsExport = dto.NfsExport ?? await _settings.GetAsync(SettingsService.NfsExport);
|
|
var nfsExportFolder = dto.NfsExportFolder ?? await _settings.GetAsync(SettingsService.NfsExportFolder);
|
|
var nfsOptions = dto.NfsExtraOptions ?? await _settings.GetAsync(SettingsService.NfsOptions, string.Empty);
|
|
|
|
var cmsImage = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3");
|
|
var newtImage = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt");
|
|
var memcachedImage = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine");
|
|
var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart");
|
|
|
|
var phpPostMaxSize = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G");
|
|
var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G");
|
|
var phpMaxExecutionTime = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600");
|
|
|
|
var repoUrl = dto.TemplateRepoUrl ?? await _settings.GetAsync(SettingsService.GitRepoUrl);
|
|
var repoPat = dto.TemplateRepoPat ?? await _settings.GetAsync(SettingsService.GitRepoPat);
|
|
|
|
if (string.IsNullOrWhiteSpace(repoUrl))
|
|
throw new InvalidOperationException("Git template repository URL is not configured. Set it in Settings → Git Repo URL.");
|
|
|
|
var templateConfig = await _git.FetchAsync(repoUrl, repoPat);
|
|
|
|
var renderCtx = new RenderContext
|
|
{
|
|
CustomerName = customerName,
|
|
CustomerAbbrev = abbrev,
|
|
StackName = stackName,
|
|
CmsServerName = cmsServerName,
|
|
HostHttpPort = hostHttpPort,
|
|
CmsImage = cmsImage,
|
|
MemcachedImage = memcachedImage,
|
|
QuickChartImage = quickChartImage,
|
|
NewtImage = newtImage,
|
|
ThemeHostPath = themePath,
|
|
MySqlHost = mySqlHost,
|
|
MySqlPort = mySqlPort,
|
|
MySqlDatabase = mySqlDbName,
|
|
MySqlUser = mySqlUser,
|
|
MySqlPassword = mySqlPassword,
|
|
SmtpServer = smtpServer,
|
|
SmtpUsername = smtpUsername,
|
|
SmtpPassword = smtpPassword,
|
|
SmtpUseTls = smtpUseTls,
|
|
SmtpUseStartTls = smtpUseStartTls,
|
|
SmtpRewriteDomain = smtpRewriteDomain,
|
|
SmtpHostname = smtpHostname,
|
|
SmtpFromLineOverride = smtpFromLineOverride,
|
|
PhpPostMaxSize = phpPostMaxSize,
|
|
PhpUploadMaxFilesize = phpUploadMaxFilesize,
|
|
PhpMaxExecutionTime = phpMaxExecutionTime,
|
|
PangolinEndpoint = pangolinEndpoint,
|
|
NewtId = dto.NewtId,
|
|
NewtSecret = dto.NewtSecret,
|
|
NfsServer = nfsServer,
|
|
NfsExport = nfsExport,
|
|
NfsExportFolder = nfsExportFolder,
|
|
NfsExtraOptions = nfsOptions,
|
|
};
|
|
|
|
_logger.LogInformation("NFS render values (update): server={NfsServer}, export={NfsExport}, folder={NfsExportFolder}",
|
|
nfsServer, nfsExport, nfsExportFolder);
|
|
|
|
var composeYaml = _compose.Render(templateConfig.Yaml, renderCtx);
|
|
|
|
if (_dockerOptions.ValidateBeforeDeploy)
|
|
{
|
|
var validationResult = _validation.Validate(composeYaml, abbrev);
|
|
if (!validationResult.IsValid)
|
|
throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}");
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(themePath))
|
|
await _docker.EnsureDirectoryAsync(themePath);
|
|
|
|
if (!string.IsNullOrWhiteSpace(nfsServer) && !string.IsNullOrWhiteSpace(nfsExport))
|
|
{
|
|
var nfsFolders = ComposeRenderService.ExtractNfsDeviceFolders(composeYaml, nfsExport, nfsExportFolder);
|
|
_logger.LogInformation("Ensuring NFS export folders on {Server}:{Export} — {Folders}",
|
|
nfsServer, nfsExport, string.Join(", ", nfsFolders));
|
|
var (nfsFoldersOk, nfsError) = await _docker.EnsureNfsFoldersWithErrorAsync(nfsServer, nfsExport, nfsFolders, nfsExportFolder);
|
|
if (!nfsFoldersOk)
|
|
throw new InvalidOperationException(
|
|
$"Failed to create NFS volume directories on {nfsServer}:{nfsExport}: {nfsError}\n"
|
|
+ "Common causes: (1) SSH user needs passwordless sudo (NOPASSWD) for mount/umount/mkdir, "
|
|
+ "(2) NFS export has root_squash enabled — set 'No mapping' / no_root_squash on the NFS server.");
|
|
}
|
|
|
|
_logger.LogInformation("Removing stale NFS volumes for stack {StackName}", stackName);
|
|
await _docker.RemoveStackVolumesAsync(stackName);
|
|
|
|
var deployResult = await _docker.DeployStackAsync(stackName, composeYaml, resolveImage: true);
|
|
if (!deployResult.Success)
|
|
throw new InvalidOperationException($"Stack redeploy failed: {deployResult.ErrorMessage}");
|
|
|
|
sw.Stop();
|
|
opLog.Status = OperationStatus.Success;
|
|
opLog.Message = $"Instance updated: {stackName}";
|
|
opLog.DurationMs = sw.ElapsedMilliseconds;
|
|
_db.OperationLogs.Add(opLog);
|
|
await _db.SaveChangesAsync();
|
|
|
|
deployResult.ServiceCount = 4;
|
|
deployResult.Message = "Instance updated and redeployed.";
|
|
return deployResult;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
sw.Stop();
|
|
opLog.Status = OperationStatus.Failure;
|
|
opLog.Message = $"Update failed: {ex.Message}";
|
|
opLog.DurationMs = sw.ElapsedMilliseconds;
|
|
_db.OperationLogs.Add(opLog);
|
|
await _db.SaveChangesAsync();
|
|
_logger.LogError(ex, "Instance update failed: {StackName}", stackName);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Delete
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
|
|
public async Task<DeploymentResultDto> DeleteInstanceAsync(
|
|
string stackName,
|
|
string customerAbbrev,
|
|
bool retainSecrets = false,
|
|
bool clearXiboCreds = true,
|
|
string? userId = null)
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
var opLog = StartOperation(OperationType.Delete, userId, stackName);
|
|
|
|
try
|
|
{
|
|
var abbrev = customerAbbrev.Trim().ToLowerInvariant();
|
|
|
|
_logger.LogInformation("Deleting instance: {StackName} (abbrev={Abbrev}) retainSecrets={RetainSecrets}",
|
|
stackName, abbrev, retainSecrets);
|
|
|
|
var result = await _docker.RemoveStackAsync(stackName);
|
|
|
|
if (!retainSecrets)
|
|
{
|
|
_logger.LogInformation("Dropping MySQL database and user for instance {StackName}", stackName);
|
|
var (dropOk, dropMsg) = await DropMySqlDatabaseAsync(abbrev);
|
|
if (!dropOk)
|
|
_logger.LogWarning("MySQL cleanup incomplete for {StackName}: {Message}", stackName, dropMsg);
|
|
else
|
|
_logger.LogInformation("MySQL cleanup complete: {Message}", dropMsg);
|
|
|
|
foreach (var secretName in AllCustomerMysqlSecretNames(abbrev))
|
|
await _secrets.DeleteSecretAsync(secretName);
|
|
|
|
// Remove the stored password from local settings
|
|
await _settings.SetAsync(
|
|
SettingsService.InstanceMySqlPassword(abbrev), null,
|
|
SettingsService.CatInstance, isSensitive: false);
|
|
await _db.SaveChangesAsync();
|
|
}
|
|
|
|
sw.Stop();
|
|
opLog.Status = OperationStatus.Success;
|
|
opLog.Message = $"Instance deleted: {stackName}";
|
|
opLog.DurationMs = sw.ElapsedMilliseconds;
|
|
_db.OperationLogs.Add(opLog);
|
|
await _db.SaveChangesAsync();
|
|
|
|
result.Message = "Instance deleted.";
|
|
return result;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
sw.Stop();
|
|
opLog.Status = OperationStatus.Failure;
|
|
opLog.Message = $"Delete failed: {ex.Message}";
|
|
opLog.DurationMs = sw.ElapsedMilliseconds;
|
|
_db.OperationLogs.Add(opLog);
|
|
await _db.SaveChangesAsync();
|
|
_logger.LogError(ex, "Instance delete failed: {StackName}", stackName);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// MySQL password rotation
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Rotates the MySQL password for an instance: generates a new password, updates the
|
|
/// Docker Swarm secret, updates the MySQL user via ALTER USER, then redeploys the stack.
|
|
/// </summary>
|
|
public async Task<(bool Success, string Message)> RotateMySqlPasswordAsync(string stackName, string? userId = null)
|
|
{
|
|
var abbrev = ExtractAbbrev(stackName);
|
|
var newPassword = GenerateRandomPassword(32);
|
|
|
|
_logger.LogInformation("Rotating MySQL password for instance {StackName}", stackName);
|
|
|
|
// ── Step 1: Update MySQL password via SSH tunnel ─────────────────────
|
|
var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
|
var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
|
|
var mySqlAdminUser = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
|
|
var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
|
|
var mySqlUser = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);
|
|
|
|
if (!int.TryParse(mySqlPort, out var port)) port = 3306;
|
|
|
|
var (mysqlOk, mysqlErr) = await _docker.AlterMySqlUserPasswordAsync(
|
|
mySqlHost, port, mySqlAdminUser, mySqlAdminPassword, mySqlUser, newPassword);
|
|
|
|
if (!mysqlOk)
|
|
return (false, $"MySQL ALTER USER failed (no service disruption): {mysqlErr}");
|
|
|
|
_logger.LogInformation("MySQL password updated for user {User}", mySqlUser);
|
|
|
|
// ── Step 2: Persist new password in settings (encrypted) ─────────────
|
|
await _settings.SetAsync(
|
|
SettingsService.InstanceMySqlPassword(abbrev), newPassword,
|
|
SettingsService.CatInstance, isSensitive: true);
|
|
await _db.SaveChangesAsync();
|
|
_logger.LogInformation("New MySQL password stored in settings for instance {Abbrev}", abbrev);
|
|
|
|
// ── Step 3: Force-update the web service to pick up new compose env ──
|
|
// Redeploy the stack so the MYSQL_PASSWORD env var gets the new value.
|
|
var webService = $"{stackName}_{abbrev}-web";
|
|
if (!await _docker.ForceUpdateServiceAsync(webService))
|
|
{
|
|
_logger.LogWarning(
|
|
"Force-update of service {Service} failed. MySQL has the new password but " +
|
|
"the container is still running with the old value. A manual redeploy may be needed.",
|
|
webService);
|
|
return (true, $"MySQL password rotated for '{stackName}' but service force-update failed. Redeploy to apply.");
|
|
}
|
|
|
|
_logger.LogInformation("Service {Service} force-updated with new password", webService);
|
|
return (true, $"MySQL password rotated successfully for '{stackName}'.");
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// MySQL database helpers
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Creates MySQL database and user on the external MySQL server via SSH tunnel.
|
|
/// </summary>
|
|
public async Task<(bool Success, string Message)> CreateMySqlDatabaseAsync(
|
|
string abbrev,
|
|
string mysqlPassword)
|
|
{
|
|
var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
|
var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
|
|
var mySqlAdminUser = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
|
|
var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
|
|
|
|
var dbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
|
|
var userName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);
|
|
|
|
_logger.LogInformation("Creating MySQL database {Db} and user {User} via SSH tunnel", dbName, userName);
|
|
|
|
if (!int.TryParse(mySqlPort, out var port))
|
|
port = 3306;
|
|
|
|
try
|
|
{
|
|
var (connection, tunnel) = await _docker.OpenMySqlConnectionAsync(
|
|
mySqlHost, port, mySqlAdminUser, mySqlAdminPassword);
|
|
await using var _ = connection;
|
|
using var __ = tunnel;
|
|
|
|
var escapedDb = dbName.Replace("`", "``");
|
|
var escapedUser = userName.Replace("'", "''");
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = $"CREATE DATABASE IF NOT EXISTS `{escapedDb}`";
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = $"CREATE USER IF NOT EXISTS '{escapedUser}'@'%' IDENTIFIED BY @pwd";
|
|
cmd.Parameters.AddWithValue("@pwd", mysqlPassword);
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = $"ALTER USER '{escapedUser}'@'%' IDENTIFIED BY @pwd";
|
|
cmd.Parameters.AddWithValue("@pwd", mysqlPassword);
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = $"GRANT ALL PRIVILEGES ON `{escapedDb}`.* TO '{escapedUser}'@'%'";
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = "FLUSH PRIVILEGES";
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
// Verify the new user's credentials by opening a connection as that user
|
|
try
|
|
{
|
|
var parsed = new MySqlConnectionStringBuilder(connection.ConnectionString);
|
|
var localPort = (int)parsed.Port;
|
|
|
|
var testCsb = new MySqlConnectionStringBuilder
|
|
{
|
|
Server = "127.0.0.1",
|
|
Port = (uint)localPort,
|
|
UserID = userName,
|
|
Password = mysqlPassword,
|
|
ConnectionTimeout = 5,
|
|
SslMode = MySqlSslMode.Disabled,
|
|
};
|
|
|
|
await using var testConn = new MySqlConnection(testCsb.ConnectionString);
|
|
await testConn.OpenAsync();
|
|
await testConn.CloseAsync();
|
|
}
|
|
catch (MySqlException ex)
|
|
{
|
|
_logger.LogError(ex, "Verification login failed for new MySQL user {User}", userName);
|
|
return (false, $"Verification login failed for user '{userName}': {ex.Message}");
|
|
}
|
|
|
|
_logger.LogInformation("MySQL database {Db} and user {User} created and verified successfully", dbName, userName);
|
|
return (true, $"Database '{dbName}' and user '{userName}' created.");
|
|
}
|
|
catch (MySqlException ex)
|
|
{
|
|
_logger.LogError(ex, "MySQL setup failed for database {Db}", dbName);
|
|
return (false, $"MySQL setup failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Drops the MySQL database and user for a given instance abbreviation.
|
|
/// </summary>
|
|
public async Task<(bool Success, string Message)> DropMySqlDatabaseAsync(string abbrev)
|
|
{
|
|
var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
|
var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
|
|
var mySqlAdminUser = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
|
|
var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
|
|
|
|
var dbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
|
|
var userName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);
|
|
|
|
_logger.LogInformation("Dropping MySQL database {Db} and user {User} via SSH tunnel", dbName, userName);
|
|
|
|
if (!int.TryParse(mySqlPort, out var port))
|
|
port = 3306;
|
|
|
|
try
|
|
{
|
|
var (connection, tunnel) = await _docker.OpenMySqlConnectionAsync(
|
|
mySqlHost, port, mySqlAdminUser, mySqlAdminPassword);
|
|
await using var _ = connection;
|
|
using var __ = tunnel;
|
|
|
|
var escapedDb = dbName.Replace("`", "``");
|
|
var escapedUser = userName.Replace("'", "''");
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = $"DROP DATABASE IF EXISTS `{escapedDb}`";
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = $"DROP USER IF EXISTS '{escapedUser}'@'%'";
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await using (var cmd = connection.CreateCommand())
|
|
{
|
|
cmd.CommandText = "FLUSH PRIVILEGES";
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
_logger.LogInformation("MySQL database {Db} and user {User} dropped successfully", dbName, userName);
|
|
return (true, $"Database '{dbName}' and user '{userName}' dropped.");
|
|
}
|
|
catch (MySqlException ex)
|
|
{
|
|
_logger.LogError(ex, "MySQL drop failed for database {Db}", dbName);
|
|
return (false, $"MySQL drop failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Private helpers
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
|
|
private static OperationLog StartOperation(OperationType type, string? userId, string? stackName = null)
|
|
=> new OperationLog { Operation = type, UserId = userId, Status = OperationStatus.Pending, StackName = stackName };
|
|
|
|
private static string GenerateRandomPassword(int length)
|
|
{
|
|
const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
|
|
return RandomNumberGenerator.GetString(chars, length);
|
|
}
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Orchestrator service for CMS instance lifecycle (Create, Update, Delete, List, Inspect).
|