using System.Diagnostics; using System.Security.Cryptography; using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using OTSSignsOrchestrator.Core.Configuration; using OTSSignsOrchestrator.Core.Data; using OTSSignsOrchestrator.Core.Models.DTOs; using OTSSignsOrchestrator.Core.Models.Entities; namespace OTSSignsOrchestrator.Core.Services; /// /// Orchestrator service for CMS instance lifecycle (Create, Update, Delete, List, Inspect). /// New‐instance flow: /// 1. Clone template repo to local cache /// 2. Generate MySQL password → create Docker Swarm secret (never persisted locally) /// 3. Create MySQL database + user on external MySQL server via SSH /// 4. Render combined compose YAML (no MySQL container, CIFS volumes, Newt service) /// 5. Deploy stack via SSH /// public class InstanceService { private readonly XiboContext _db; private readonly GitTemplateService _git; private readonly ComposeRenderService _compose; private readonly ComposeValidationService _validation; private readonly IDockerCliService _docker; private readonly IDockerSecretsService _secrets; private readonly XiboApiService _xibo; private readonly SettingsService _settings; private readonly DockerOptions _dockerOptions; private readonly ILogger _logger; public InstanceService( XiboContext db, GitTemplateService git, ComposeRenderService compose, ComposeValidationService validation, IDockerCliService docker, IDockerSecretsService secrets, XiboApiService xibo, SettingsService settings, IOptions dockerOptions, ILogger logger) { _db = db; _git = git; _compose = compose; _validation = validation; _docker = docker; _secrets = secrets; _xibo = xibo; _settings = settings; _dockerOptions = dockerOptions.Value; _logger = logger; } /// /// Creates a new CMS instance: /// 1. Clone repo 2. Generate secrets 3. Create MySQL DB/user 4. Render compose 5. Deploy /// public async Task CreateInstanceAsync(CreateInstanceDto dto, string? userId = null) { var sw = Stopwatch.StartNew(); var opLog = StartOperation(OperationType.Create, userId); var abbrev = dto.CustomerAbbrev.Trim().ToLowerInvariant(); var stackName = $"{abbrev}-cms-stack"; try { _logger.LogInformation("Creating instance: abbrev={Abbrev}, customer={Customer}", abbrev, dto.CustomerName); // ── Check uniqueness ──────────────────────────────────────────── var existing = await _db.CmsInstances.IgnoreQueryFilters() .FirstOrDefaultAsync(i => i.StackName == stackName && i.DeletedAt == null); if (existing != null) throw new InvalidOperationException($"Stack '{stackName}' already exists."); // ── 1. Clone template repo (optional) ─────────────────────────── var repoUrl = await _settings.GetAsync(SettingsService.GitRepoUrl); var repoPat = await _settings.GetAsync(SettingsService.GitRepoPat); if (!string.IsNullOrWhiteSpace(repoUrl)) { _logger.LogInformation("Cloning template repo: {RepoUrl}", repoUrl); await _git.FetchAsync(repoUrl, repoPat); } // ── 2. Generate MySQL password → Docker Swarm secret ──────────── var mysqlPassword = GenerateRandomPassword(32); var mysqlSecretName = $"{abbrev}-cms-db-password"; await _secrets.EnsureSecretAsync(mysqlSecretName, mysqlPassword); await EnsureSecretMetadata(mysqlSecretName, false, dto.CustomerName); _logger.LogInformation("Docker secret created: {SecretName}", mysqlSecretName); // ── 3. Read settings ──────────────────────────────────────────── var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost"); var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306"); var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev); var mySqlUser = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms")).Replace("{abbrev}", abbrev); var cmsServerName = (await _settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "{abbrev}.ots-signs.com")).Replace("{abbrev}", abbrev); var themePath = (await _settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme")).Replace("{abbrev}", abbrev); var smtpServer = await _settings.GetAsync(SettingsService.SmtpServer, string.Empty); var smtpUsername = await _settings.GetAsync(SettingsService.SmtpUsername, string.Empty); var smtpPassword = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty); var smtpUseTls = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES"); var smtpUseStartTls = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES"); var smtpRewriteDomain = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty); var smtpHostname = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty); var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO"); var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net"); var cifsServer = dto.CifsServer ?? await _settings.GetAsync(SettingsService.CifsServer); var cifsShareBasePath = dto.CifsShareBasePath ?? await _settings.GetAsync(SettingsService.CifsShareBasePath); var cifsUsername = dto.CifsUsername ?? await _settings.GetAsync(SettingsService.CifsUsername); var cifsPassword = dto.CifsPassword ?? await _settings.GetAsync(SettingsService.CifsPassword); var cifsOptions = dto.CifsExtraOptions ?? await _settings.GetAsync(SettingsService.CifsOptions, "file_mode=0777,dir_mode=0777"); var cmsImage = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3"); var newtImage = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt"); var memcachedImage = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine"); var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart"); var phpPostMaxSize = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G"); var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G"); var phpMaxExecutionTime = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600"); // ── 4. Render compose YAML ────────────────────────────────────── var renderCtx = new RenderContext { CustomerName = dto.CustomerName, CustomerAbbrev = abbrev, StackName = stackName, CmsServerName = cmsServerName, HostHttpPort = 80, CmsImage = cmsImage, MemcachedImage = memcachedImage, QuickChartImage = quickChartImage, NewtImage = newtImage, ThemeHostPath = themePath, MySqlHost = mySqlHost, MySqlPort = mySqlPort, MySqlDatabase = mySqlDbName, MySqlUser = mySqlUser, SmtpServer = smtpServer, SmtpUsername = smtpUsername, SmtpPassword = smtpPassword, SmtpUseTls = smtpUseTls, SmtpUseStartTls = smtpUseStartTls, SmtpRewriteDomain = smtpRewriteDomain, SmtpHostname = smtpHostname, SmtpFromLineOverride = smtpFromLineOverride, PhpPostMaxSize = phpPostMaxSize, PhpUploadMaxFilesize = phpUploadMaxFilesize, PhpMaxExecutionTime = phpMaxExecutionTime, IncludeNewt = true, PangolinEndpoint = pangolinEndpoint, NewtId = dto.NewtId, NewtSecret = dto.NewtSecret, UseCifsVolumes = !string.IsNullOrWhiteSpace(cifsServer), CifsServer = cifsServer, CifsShareBasePath = cifsShareBasePath, CifsUsername = cifsUsername, CifsPassword = cifsPassword, CifsExtraOptions = cifsOptions, SecretNames = new List { mysqlSecretName }, }; var composeYaml = _compose.Render(renderCtx); if (_dockerOptions.ValidateBeforeDeploy) { var validationResult = _validation.Validate(composeYaml, abbrev); if (!validationResult.IsValid) throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}"); } // ── 5. Deploy stack ───────────────────────────────────────────── var deployResult = await _docker.DeployStackAsync(stackName, composeYaml); if (!deployResult.Success) throw new InvalidOperationException($"Stack deployment failed: {deployResult.ErrorMessage}"); // ── 6. Record instance ────────────────────────────────────────── var instance = new CmsInstance { CustomerName = dto.CustomerName, CustomerAbbrev = abbrev, StackName = stackName, CmsServerName = cmsServerName, HostHttpPort = 80, ThemeHostPath = themePath, LibraryHostPath = $"{abbrev}-cms-library", SmtpServer = smtpServer, SmtpUsername = smtpUsername, TemplateRepoUrl = repoUrl ?? string.Empty, TemplateRepoPat = repoPat, Status = InstanceStatus.Active, SshHostId = dto.SshHostId, CifsServer = cifsServer, CifsShareBasePath = cifsShareBasePath, CifsUsername = cifsUsername, CifsPassword = cifsPassword, CifsExtraOptions = cifsOptions, }; _db.CmsInstances.Add(instance); sw.Stop(); opLog.InstanceId = instance.Id; opLog.Status = OperationStatus.Success; opLog.Message = $"Instance deployed: {stackName}"; opLog.DurationMs = sw.ElapsedMilliseconds; _db.OperationLogs.Add(opLog); await _db.SaveChangesAsync(); _logger.LogInformation("Instance created: {StackName} (id={Id}) | duration={DurationMs}ms", stackName, instance.Id, sw.ElapsedMilliseconds); deployResult.ServiceCount = renderCtx.IncludeNewt ? 4 : 3; deployResult.Message = "Instance deployed successfully."; return deployResult; } catch (Exception ex) { sw.Stop(); opLog.Status = OperationStatus.Failure; opLog.Message = $"Create failed: {ex.Message}"; opLog.DurationMs = sw.ElapsedMilliseconds; _db.OperationLogs.Add(opLog); await _db.SaveChangesAsync(); _logger.LogError(ex, "Instance create failed: {StackName}", stackName); throw; } } /// /// Creates MySQL database and user on external MySQL server via SSH. /// Called by the ViewModel before CreateInstanceAsync since it needs SSH access. /// public async Task<(bool Success, string Message)> CreateMySqlDatabaseAsync( string abbrev, string mysqlPassword, Func> runSshCommand) { var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost"); var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306"); var mySqlAdminUser = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root"); var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty); var dbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev); var userName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms")).Replace("{abbrev}", abbrev); var safePwd = mySqlAdminPassword.Replace("'", "'\\''"); var safeUserPwd = mysqlPassword.Replace("'", "'\\''"); var sql = $"CREATE DATABASE IF NOT EXISTS \\`{dbName}\\`; " + $"CREATE USER IF NOT EXISTS '{userName}'@'%' IDENTIFIED BY '{safeUserPwd}'; " + $"GRANT ALL PRIVILEGES ON \\`{dbName}\\`.* TO '{userName}'@'%'; " + $"FLUSH PRIVILEGES;"; var cmd = $"mysql -h {mySqlHost} -P {mySqlPort} -u {mySqlAdminUser} -p'{safePwd}' -e \"{sql}\" 2>&1"; _logger.LogInformation("Creating MySQL database {Db} and user {User}", dbName, userName); var (exitCode, stdout, stderr) = await runSshCommand(cmd); if (exitCode == 0) { _logger.LogInformation("MySQL database and user created: {Db} / {User}", dbName, userName); return (true, $"Database '{dbName}' and user '{userName}' created."); } var error = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr; _logger.LogError("MySQL setup failed: {Error}", error); return (false, $"MySQL setup failed: {error.Trim()}"); } public async Task UpdateInstanceAsync(Guid id, UpdateInstanceDto dto, string? userId = null) { var sw = Stopwatch.StartNew(); var opLog = StartOperation(OperationType.Update, userId); try { var instance = await _db.CmsInstances.FindAsync(id) ?? throw new KeyNotFoundException($"Instance {id} not found."); _logger.LogInformation("Updating instance: {StackName} (id={Id})", instance.StackName, id); if (dto.TemplateRepoUrl != null) instance.TemplateRepoUrl = dto.TemplateRepoUrl; if (dto.TemplateRepoPat != null) instance.TemplateRepoPat = dto.TemplateRepoPat; if (dto.SmtpServer != null) instance.SmtpServer = dto.SmtpServer; if (dto.SmtpUsername != null) instance.SmtpUsername = dto.SmtpUsername; if (dto.Constraints != null) instance.Constraints = JsonSerializer.Serialize(dto.Constraints); if (dto.XiboUsername != null) instance.XiboUsername = dto.XiboUsername; if (dto.XiboPassword != null) instance.XiboPassword = dto.XiboPassword; if (dto.CifsServer != null) instance.CifsServer = dto.CifsServer; if (dto.CifsShareBasePath != null) instance.CifsShareBasePath = dto.CifsShareBasePath; if (dto.CifsUsername != null) instance.CifsUsername = dto.CifsUsername; if (dto.CifsPassword != null) instance.CifsPassword = dto.CifsPassword; if (dto.CifsExtraOptions != null) instance.CifsExtraOptions = dto.CifsExtraOptions; var abbrev = instance.CustomerAbbrev; var mysqlSecretName = $"{abbrev}-cms-db-password"; // Read current settings for re-render var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost"); var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306"); var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev); var mySqlUser = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms")).Replace("{abbrev}", abbrev); var smtpServer = instance.SmtpServer; var smtpUsername = instance.SmtpUsername; var smtpPassword = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty); var smtpUseTls = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES"); var smtpUseStartTls = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES"); var smtpRewriteDomain = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty); var smtpHostname = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty); var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO"); var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net"); // Use per-instance CIFS credentials var cifsServer = instance.CifsServer; var cifsShareBasePath = instance.CifsShareBasePath; var cifsUsername = instance.CifsUsername; var cifsPassword = instance.CifsPassword; var cifsOptions = instance.CifsExtraOptions ?? "file_mode=0777,dir_mode=0777"; var cmsImage = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3"); var newtImage = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt"); var memcachedImage = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine"); var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart"); var phpPostMaxSize = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G"); var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G"); var phpMaxExecutionTime = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600"); var renderCtx = new RenderContext { CustomerName = instance.CustomerName, CustomerAbbrev = abbrev, StackName = instance.StackName, CmsServerName = instance.CmsServerName, HostHttpPort = instance.HostHttpPort, CmsImage = cmsImage, MemcachedImage = memcachedImage, QuickChartImage = quickChartImage, NewtImage = newtImage, ThemeHostPath = instance.ThemeHostPath, MySqlHost = mySqlHost, MySqlPort = mySqlPort, MySqlDatabase = mySqlDbName, MySqlUser = mySqlUser, SmtpServer = smtpServer, SmtpUsername = smtpUsername, SmtpPassword = smtpPassword, SmtpUseTls = smtpUseTls, SmtpUseStartTls = smtpUseStartTls, SmtpRewriteDomain = smtpRewriteDomain, SmtpHostname = smtpHostname, SmtpFromLineOverride = smtpFromLineOverride, PhpPostMaxSize = phpPostMaxSize, PhpUploadMaxFilesize = phpUploadMaxFilesize, PhpMaxExecutionTime = phpMaxExecutionTime, IncludeNewt = true, PangolinEndpoint = pangolinEndpoint, UseCifsVolumes = !string.IsNullOrWhiteSpace(cifsServer), CifsServer = cifsServer, CifsShareBasePath = cifsShareBasePath, CifsUsername = cifsUsername, CifsPassword = cifsPassword, CifsExtraOptions = cifsOptions, SecretNames = new List { mysqlSecretName }, }; var composeYaml = _compose.Render(renderCtx); if (_dockerOptions.ValidateBeforeDeploy) { var validationResult = _validation.Validate(composeYaml, abbrev); if (!validationResult.IsValid) throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}"); } var deployResult = await _docker.DeployStackAsync(instance.StackName, composeYaml, resolveImage: true); if (!deployResult.Success) throw new InvalidOperationException($"Stack redeploy failed: {deployResult.ErrorMessage}"); instance.UpdatedAt = DateTime.UtcNow; instance.Status = InstanceStatus.Active; sw.Stop(); opLog.InstanceId = instance.Id; opLog.Status = OperationStatus.Success; opLog.Message = $"Instance updated: {instance.StackName}"; opLog.DurationMs = sw.ElapsedMilliseconds; _db.OperationLogs.Add(opLog); await _db.SaveChangesAsync(); deployResult.ServiceCount = 4; deployResult.Message = "Instance updated and redeployed."; return deployResult; } catch (Exception ex) { sw.Stop(); opLog.Status = OperationStatus.Failure; opLog.Message = $"Update failed: {ex.Message}"; opLog.DurationMs = sw.ElapsedMilliseconds; _db.OperationLogs.Add(opLog); await _db.SaveChangesAsync(); _logger.LogError(ex, "Instance update failed (id={Id})", id); throw; } } public async Task DeleteInstanceAsync( Guid id, bool retainSecrets = false, bool clearXiboCreds = true, string? userId = null) { var sw = Stopwatch.StartNew(); var opLog = StartOperation(OperationType.Delete, userId); try { var instance = await _db.CmsInstances.FindAsync(id) ?? throw new KeyNotFoundException($"Instance {id} not found."); _logger.LogInformation("Deleting instance: {StackName} (id={Id}) retainSecrets={RetainSecrets}", instance.StackName, id, retainSecrets); var result = await _docker.RemoveStackAsync(instance.StackName); if (!retainSecrets) { var mysqlSecretName = $"{instance.CustomerAbbrev}-cms-db-password"; await _secrets.DeleteSecretAsync(mysqlSecretName); var secretMeta = await _db.SecretMetadata.FirstOrDefaultAsync(s => s.Name == mysqlSecretName); if (secretMeta != null) _db.SecretMetadata.Remove(secretMeta); } instance.Status = InstanceStatus.Deleted; instance.DeletedAt = DateTime.UtcNow; instance.UpdatedAt = DateTime.UtcNow; if (clearXiboCreds) { instance.XiboUsername = null; instance.XiboPassword = null; instance.XiboApiTestStatus = XiboApiTestStatus.Unknown; } sw.Stop(); opLog.InstanceId = instance.Id; opLog.Status = OperationStatus.Success; opLog.Message = $"Instance deleted: {instance.StackName}"; opLog.DurationMs = sw.ElapsedMilliseconds; _db.OperationLogs.Add(opLog); await _db.SaveChangesAsync(); result.Message = "Instance deleted."; return result; } catch (Exception ex) { sw.Stop(); opLog.Status = OperationStatus.Failure; opLog.Message = $"Delete failed: {ex.Message}"; opLog.DurationMs = sw.ElapsedMilliseconds; _db.OperationLogs.Add(opLog); await _db.SaveChangesAsync(); _logger.LogError(ex, "Instance delete failed (id={Id})", id); throw; } } public async Task GetInstanceAsync(Guid id) => await _db.CmsInstances.Include(i => i.SshHost).FirstOrDefaultAsync(i => i.Id == id); public async Task<(List Items, int TotalCount)> ListInstancesAsync( int page = 1, int pageSize = 50, string? filter = null) { var query = _db.CmsInstances.Include(i => i.SshHost).AsQueryable(); if (!string.IsNullOrWhiteSpace(filter)) query = query.Where(i => i.CustomerName.Contains(filter) || i.StackName.Contains(filter)); var total = await query.CountAsync(); var items = await query.OrderByDescending(i => i.CreatedAt) .Skip((page - 1) * pageSize).Take(pageSize).ToListAsync(); return (items, total); } public async Task TestXiboConnectionAsync(Guid id) { var instance = await _db.CmsInstances.FindAsync(id) ?? throw new KeyNotFoundException($"Instance {id} not found."); if (string.IsNullOrEmpty(instance.XiboUsername) || string.IsNullOrEmpty(instance.XiboPassword)) return new XiboTestResult { IsValid = false, Message = "No Xibo credentials stored." }; var url = $"http://localhost:{instance.HostHttpPort}"; var result = await _xibo.TestConnectionAsync(url, instance.XiboUsername, instance.XiboPassword); instance.XiboApiTestStatus = result.IsValid ? XiboApiTestStatus.Success : XiboApiTestStatus.Failed; instance.XiboApiTestedAt = DateTime.UtcNow; await _db.SaveChangesAsync(); return result; } private async Task EnsureSecretMetadata(string name, bool isGlobal, string? customerName) { var existing = await _db.SecretMetadata.FirstOrDefaultAsync(s => s.Name == name); if (existing == null) { _db.SecretMetadata.Add(new SecretMetadata { Name = name, IsGlobal = isGlobal, CustomerName = customerName }); } } private static OperationLog StartOperation(OperationType type, string? userId) => new OperationLog { Operation = type, UserId = userId, Status = OperationStatus.Pending }; private static string GenerateRandomPassword(int length) { const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*"; return RandomNumberGenerator.GetString(chars, length); } }