OTSSignsOrchestrator/OTSSignsOrchestrator.Core/Services/InstanceService.cs

using System.Diagnostics;
using System.Security.Cryptography;
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using OTSSignsOrchestrator.Core.Configuration;
using OTSSignsOrchestrator.Core.Data;
using OTSSignsOrchestrator.Core.Models.DTOs;
using OTSSignsOrchestrator.Core.Models.Entities;

namespace OTSSignsOrchestrator.Core.Services;

/// <summary>
/// Orchestrator service for CMS instance lifecycle (Create, Update, Delete, List, Inspect).
/// New‐instance flow:
///   1. Clone template repo to local cache
///   2. Generate MySQL password → create Docker Swarm secret (never persisted locally)
///   3. Create MySQL database + user on external MySQL server via SSH
///   4. Render combined compose YAML (no MySQL container, CIFS volumes, Newt service)
///   5. Deploy stack via SSH
/// </summary>
public class InstanceService
{
    private readonly XiboContext _db;
    private readonly GitTemplateService _git;
    private readonly ComposeRenderService _compose;
    private readonly ComposeValidationService _validation;
    private readonly IDockerCliService _docker;
    private readonly IDockerSecretsService _secrets;
    private readonly XiboApiService _xibo;
    private readonly SettingsService _settings;
    private readonly DockerOptions _dockerOptions;
    private readonly ILogger<InstanceService> _logger;

    public InstanceService(
        XiboContext db,
        GitTemplateService git,
        ComposeRenderService compose,
        ComposeValidationService validation,
        IDockerCliService docker,
        IDockerSecretsService secrets,
        XiboApiService xibo,
        SettingsService settings,
        IOptions<DockerOptions> dockerOptions,
        ILogger<InstanceService> logger)
    {
        _db = db;
        _git = git;
        _compose = compose;
        _validation = validation;
        _docker = docker;
        _secrets = secrets;
        _xibo = xibo;
        _settings = settings;
        _dockerOptions = dockerOptions.Value;
        _logger = logger;
    }

    /// <summary>
    /// Creates a new CMS instance:
    /// 1. Clone repo  2. Generate secrets  3. Create MySQL DB/user  4. Render compose  5. Deploy
    /// </summary>
    public async Task<DeploymentResultDto> CreateInstanceAsync(CreateInstanceDto dto, string? userId = null)
    {
        var sw = Stopwatch.StartNew();
        var opLog = StartOperation(OperationType.Create, userId);
        var abbrev = dto.CustomerAbbrev.Trim().ToLowerInvariant();
        var stackName = $"{abbrev}-cms-stack";

        try
        {
            _logger.LogInformation("Creating instance: abbrev={Abbrev}, customer={Customer}", abbrev, dto.CustomerName);

            // ── Check uniqueness ────────────────────────────────────────────
            var existing = await _db.CmsInstances.IgnoreQueryFilters()
                .FirstOrDefaultAsync(i => i.StackName == stackName && i.DeletedAt == null);
            if (existing != null)
                throw new InvalidOperationException($"Stack '{stackName}' already exists.");

            // ── 1. Clone template repo (optional) ───────────────────────────
            var repoUrl = await _settings.GetAsync(SettingsService.GitRepoUrl);
            var repoPat = await _settings.GetAsync(SettingsService.GitRepoPat);

            if (!string.IsNullOrWhiteSpace(repoUrl))
            {
                _logger.LogInformation("Cloning template repo: {RepoUrl}", repoUrl);
                await _git.FetchAsync(repoUrl, repoPat);
            }

            // ── 2. Generate MySQL password → Docker Swarm secret ────────────
            var mysqlPassword = GenerateRandomPassword(32);
            var mysqlSecretName = $"{abbrev}-cms-db-password";
            await _secrets.EnsureSecretAsync(mysqlSecretName, mysqlPassword);
            await EnsureSecretMetadata(mysqlSecretName, false, dto.CustomerName);
            _logger.LogInformation("Docker secret created: {SecretName}", mysqlSecretName);

            // ── 3. Read settings ────────────────────────────────────────────
            var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
            var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
            var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
            var mySqlUser = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms")).Replace("{abbrev}", abbrev);

            var cmsServerName = (await _settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "{abbrev}.ots-signs.com")).Replace("{abbrev}", abbrev);
            var themePath = (await _settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme")).Replace("{abbrev}", abbrev);

            var smtpServer = await _settings.GetAsync(SettingsService.SmtpServer, string.Empty);
            var smtpUsername = await _settings.GetAsync(SettingsService.SmtpUsername, string.Empty);
            var smtpPassword = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty);
            var smtpUseTls = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES");
            var smtpUseStartTls = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES");
            var smtpRewriteDomain = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty);
            var smtpHostname = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty);
            var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO");

            var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net");

            var cifsServer = dto.CifsServer ?? await _settings.GetAsync(SettingsService.CifsServer);
            var cifsShareBasePath = dto.CifsShareBasePath ?? await _settings.GetAsync(SettingsService.CifsShareBasePath);
            var cifsUsername = dto.CifsUsername ?? await _settings.GetAsync(SettingsService.CifsUsername);
            var cifsPassword = dto.CifsPassword ?? await _settings.GetAsync(SettingsService.CifsPassword);
            var cifsOptions = dto.CifsExtraOptions ?? await _settings.GetAsync(SettingsService.CifsOptions, "file_mode=0777,dir_mode=0777");

            var cmsImage = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3");
            var newtImage = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt");
            var memcachedImage = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine");
            var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart");

            var phpPostMaxSize = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G");
            var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G");
            var phpMaxExecutionTime = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600");

            // ── 4. Render compose YAML ──────────────────────────────────────
            var renderCtx = new RenderContext
            {
                CustomerName = dto.CustomerName,
                CustomerAbbrev = abbrev,
                StackName = stackName,
                CmsServerName = cmsServerName,
                HostHttpPort = 80,
                CmsImage = cmsImage,
                MemcachedImage = memcachedImage,
                QuickChartImage = quickChartImage,
                NewtImage = newtImage,
                ThemeHostPath = themePath,
                MySqlHost = mySqlHost,
                MySqlPort = mySqlPort,
                MySqlDatabase = mySqlDbName,
                MySqlUser = mySqlUser,
                SmtpServer = smtpServer,
                SmtpUsername = smtpUsername,
                SmtpPassword = smtpPassword,
                SmtpUseTls = smtpUseTls,
                SmtpUseStartTls = smtpUseStartTls,
                SmtpRewriteDomain = smtpRewriteDomain,
                SmtpHostname = smtpHostname,
                SmtpFromLineOverride = smtpFromLineOverride,
                PhpPostMaxSize = phpPostMaxSize,
                PhpUploadMaxFilesize = phpUploadMaxFilesize,
                PhpMaxExecutionTime = phpMaxExecutionTime,
                IncludeNewt = true,
                PangolinEndpoint = pangolinEndpoint,
                NewtId = dto.NewtId,
                NewtSecret = dto.NewtSecret,
                UseCifsVolumes = !string.IsNullOrWhiteSpace(cifsServer),
                CifsServer = cifsServer,
                CifsShareBasePath = cifsShareBasePath,
                CifsUsername = cifsUsername,
                CifsPassword = cifsPassword,
                CifsExtraOptions = cifsOptions,
                SecretNames = new List<string> { mysqlSecretName },
            };

            var composeYaml = _compose.Render(renderCtx);

            if (_dockerOptions.ValidateBeforeDeploy)
            {
                var validationResult = _validation.Validate(composeYaml, abbrev);
                if (!validationResult.IsValid)
                    throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}");
            }

            // ── 5. Deploy stack ─────────────────────────────────────────────
            var deployResult = await _docker.DeployStackAsync(stackName, composeYaml);
            if (!deployResult.Success)
                throw new InvalidOperationException($"Stack deployment failed: {deployResult.ErrorMessage}");

            // ── 6. Record instance ──────────────────────────────────────────
            var instance = new CmsInstance
            {
                CustomerName = dto.CustomerName,
                CustomerAbbrev = abbrev,
                StackName = stackName,
                CmsServerName = cmsServerName,
                HostHttpPort = 80,
                ThemeHostPath = themePath,
                LibraryHostPath = $"{abbrev}-cms-library",
                SmtpServer = smtpServer,
                SmtpUsername = smtpUsername,
                TemplateRepoUrl = repoUrl ?? string.Empty,
                TemplateRepoPat = repoPat,
                Status = InstanceStatus.Active,
                SshHostId = dto.SshHostId,
                CifsServer = cifsServer,
                CifsShareBasePath = cifsShareBasePath,
                CifsUsername = cifsUsername,
                CifsPassword = cifsPassword,
                CifsExtraOptions = cifsOptions,
            };

            _db.CmsInstances.Add(instance);
            sw.Stop();
            opLog.InstanceId = instance.Id;
            opLog.Status = OperationStatus.Success;
            opLog.Message = $"Instance deployed: {stackName}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();

            _logger.LogInformation("Instance created: {StackName} (id={Id}) | duration={DurationMs}ms",
                stackName, instance.Id, sw.ElapsedMilliseconds);

            deployResult.ServiceCount = renderCtx.IncludeNewt ? 4 : 3;
            deployResult.Message = "Instance deployed successfully.";
            return deployResult;
        }
        catch (Exception ex)
        {
            sw.Stop();
            opLog.Status = OperationStatus.Failure;
            opLog.Message = $"Create failed: {ex.Message}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();
            _logger.LogError(ex, "Instance create failed: {StackName}", stackName);
            throw;
        }
    }

    /// <summary>
    /// Creates MySQL database and user on external MySQL server via SSH.
    /// Called by the ViewModel before CreateInstanceAsync since it needs SSH access.
    /// </summary>
    public async Task<(bool Success, string Message)> CreateMySqlDatabaseAsync(
        string abbrev,
        string mysqlPassword,
        Func<string, Task<(int ExitCode, string Stdout, string Stderr)>> runSshCommand)
    {
        var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
        var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
        var mySqlAdminUser = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
        var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);

        var dbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
        var userName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms")).Replace("{abbrev}", abbrev);

        var safePwd = mySqlAdminPassword.Replace("'", "'\\''");
        var safeUserPwd = mysqlPassword.Replace("'", "'\\''");

        var sql = $"CREATE DATABASE IF NOT EXISTS \\`{dbName}\\`; "
                + $"CREATE USER IF NOT EXISTS '{userName}'@'%' IDENTIFIED BY '{safeUserPwd}'; "
                + $"GRANT ALL PRIVILEGES ON \\`{dbName}\\`.* TO '{userName}'@'%'; "
                + $"FLUSH PRIVILEGES;";

        var cmd = $"mysql -h {mySqlHost} -P {mySqlPort} -u {mySqlAdminUser} -p'{safePwd}' -e \"{sql}\" 2>&1";

        _logger.LogInformation("Creating MySQL database {Db} and user {User}", dbName, userName);

        var (exitCode, stdout, stderr) = await runSshCommand(cmd);

        if (exitCode == 0)
        {
            _logger.LogInformation("MySQL database and user created: {Db} / {User}", dbName, userName);
            return (true, $"Database '{dbName}' and user '{userName}' created.");
        }

        var error = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr;
        _logger.LogError("MySQL setup failed: {Error}", error);
        return (false, $"MySQL setup failed: {error.Trim()}");
    }

    public async Task<DeploymentResultDto> UpdateInstanceAsync(Guid id, UpdateInstanceDto dto, string? userId = null)
    {
        var sw = Stopwatch.StartNew();
        var opLog = StartOperation(OperationType.Update, userId);

        try
        {
            var instance = await _db.CmsInstances.FindAsync(id)
                ?? throw new KeyNotFoundException($"Instance {id} not found.");

            _logger.LogInformation("Updating instance: {StackName} (id={Id})", instance.StackName, id);

            if (dto.TemplateRepoUrl != null) instance.TemplateRepoUrl = dto.TemplateRepoUrl;
            if (dto.TemplateRepoPat != null) instance.TemplateRepoPat = dto.TemplateRepoPat;
            if (dto.SmtpServer != null) instance.SmtpServer = dto.SmtpServer;
            if (dto.SmtpUsername != null) instance.SmtpUsername = dto.SmtpUsername;
            if (dto.Constraints != null) instance.Constraints = JsonSerializer.Serialize(dto.Constraints);
            if (dto.XiboUsername != null) instance.XiboUsername = dto.XiboUsername;
            if (dto.XiboPassword != null) instance.XiboPassword = dto.XiboPassword;
            if (dto.CifsServer != null) instance.CifsServer = dto.CifsServer;
            if (dto.CifsShareBasePath != null) instance.CifsShareBasePath = dto.CifsShareBasePath;
            if (dto.CifsUsername != null) instance.CifsUsername = dto.CifsUsername;
            if (dto.CifsPassword != null) instance.CifsPassword = dto.CifsPassword;
            if (dto.CifsExtraOptions != null) instance.CifsExtraOptions = dto.CifsExtraOptions;

            var abbrev = instance.CustomerAbbrev;
            var mysqlSecretName = $"{abbrev}-cms-db-password";

            // Read current settings for re-render
            var mySqlHost = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
            var mySqlPort = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
            var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
            var mySqlUser = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms")).Replace("{abbrev}", abbrev);

            var smtpServer = instance.SmtpServer;
            var smtpUsername = instance.SmtpUsername;
            var smtpPassword = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty);
            var smtpUseTls = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES");
            var smtpUseStartTls = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES");
            var smtpRewriteDomain = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty);
            var smtpHostname = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty);
            var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO");

            var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net");

            // Use per-instance CIFS credentials
            var cifsServer = instance.CifsServer;
            var cifsShareBasePath = instance.CifsShareBasePath;
            var cifsUsername = instance.CifsUsername;
            var cifsPassword = instance.CifsPassword;
            var cifsOptions = instance.CifsExtraOptions ?? "file_mode=0777,dir_mode=0777";

            var cmsImage = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3");
            var newtImage = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt");
            var memcachedImage = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine");
            var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart");

            var phpPostMaxSize = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G");
            var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G");
            var phpMaxExecutionTime = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600");

            var renderCtx = new RenderContext
            {
                CustomerName = instance.CustomerName,
                CustomerAbbrev = abbrev,
                StackName = instance.StackName,
                CmsServerName = instance.CmsServerName,
                HostHttpPort = instance.HostHttpPort,
                CmsImage = cmsImage,
                MemcachedImage = memcachedImage,
                QuickChartImage = quickChartImage,
                NewtImage = newtImage,
                ThemeHostPath = instance.ThemeHostPath,
                MySqlHost = mySqlHost,
                MySqlPort = mySqlPort,
                MySqlDatabase = mySqlDbName,
                MySqlUser = mySqlUser,
                SmtpServer = smtpServer,
                SmtpUsername = smtpUsername,
                SmtpPassword = smtpPassword,
                SmtpUseTls = smtpUseTls,
                SmtpUseStartTls = smtpUseStartTls,
                SmtpRewriteDomain = smtpRewriteDomain,
                SmtpHostname = smtpHostname,
                SmtpFromLineOverride = smtpFromLineOverride,
                PhpPostMaxSize = phpPostMaxSize,
                PhpUploadMaxFilesize = phpUploadMaxFilesize,
                PhpMaxExecutionTime = phpMaxExecutionTime,
                IncludeNewt = true,
                PangolinEndpoint = pangolinEndpoint,
                UseCifsVolumes = !string.IsNullOrWhiteSpace(cifsServer),
                CifsServer = cifsServer,
                CifsShareBasePath = cifsShareBasePath,
                CifsUsername = cifsUsername,
                CifsPassword = cifsPassword,
                CifsExtraOptions = cifsOptions,
                SecretNames = new List<string> { mysqlSecretName },
            };

            var composeYaml = _compose.Render(renderCtx);

            if (_dockerOptions.ValidateBeforeDeploy)
            {
                var validationResult = _validation.Validate(composeYaml, abbrev);
                if (!validationResult.IsValid)
                    throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}");
            }

            var deployResult = await _docker.DeployStackAsync(instance.StackName, composeYaml, resolveImage: true);
            if (!deployResult.Success)
                throw new InvalidOperationException($"Stack redeploy failed: {deployResult.ErrorMessage}");

            instance.UpdatedAt = DateTime.UtcNow;
            instance.Status = InstanceStatus.Active;

            sw.Stop();
            opLog.InstanceId = instance.Id;
            opLog.Status = OperationStatus.Success;
            opLog.Message = $"Instance updated: {instance.StackName}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();

            deployResult.ServiceCount = 4;
            deployResult.Message = "Instance updated and redeployed.";
            return deployResult;
        }
        catch (Exception ex)
        {
            sw.Stop();
            opLog.Status = OperationStatus.Failure;
            opLog.Message = $"Update failed: {ex.Message}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();
            _logger.LogError(ex, "Instance update failed (id={Id})", id);
            throw;
        }
    }

    public async Task<DeploymentResultDto> DeleteInstanceAsync(
        Guid id, bool retainSecrets = false, bool clearXiboCreds = true, string? userId = null)
    {
        var sw = Stopwatch.StartNew();
        var opLog = StartOperation(OperationType.Delete, userId);

        try
        {
            var instance = await _db.CmsInstances.FindAsync(id)
                ?? throw new KeyNotFoundException($"Instance {id} not found.");

            _logger.LogInformation("Deleting instance: {StackName} (id={Id}) retainSecrets={RetainSecrets}",
                instance.StackName, id, retainSecrets);

            var result = await _docker.RemoveStackAsync(instance.StackName);

            if (!retainSecrets)
            {
                var mysqlSecretName = $"{instance.CustomerAbbrev}-cms-db-password";
                await _secrets.DeleteSecretAsync(mysqlSecretName);
                var secretMeta = await _db.SecretMetadata.FirstOrDefaultAsync(s => s.Name == mysqlSecretName);
                if (secretMeta != null)
                    _db.SecretMetadata.Remove(secretMeta);
            }

            instance.Status = InstanceStatus.Deleted;
            instance.DeletedAt = DateTime.UtcNow;
            instance.UpdatedAt = DateTime.UtcNow;

            if (clearXiboCreds)
            {
                instance.XiboUsername = null;
                instance.XiboPassword = null;
                instance.XiboApiTestStatus = XiboApiTestStatus.Unknown;
            }

            sw.Stop();
            opLog.InstanceId = instance.Id;
            opLog.Status = OperationStatus.Success;
            opLog.Message = $"Instance deleted: {instance.StackName}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();

            result.Message = "Instance deleted.";
            return result;
        }
        catch (Exception ex)
        {
            sw.Stop();
            opLog.Status = OperationStatus.Failure;
            opLog.Message = $"Delete failed: {ex.Message}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();
            _logger.LogError(ex, "Instance delete failed (id={Id})", id);
            throw;
        }
    }

    public async Task<CmsInstance?> GetInstanceAsync(Guid id)
        => await _db.CmsInstances.Include(i => i.SshHost).FirstOrDefaultAsync(i => i.Id == id);

    public async Task<(List<CmsInstance> Items, int TotalCount)> ListInstancesAsync(
        int page = 1, int pageSize = 50, string? filter = null)
    {
        var query = _db.CmsInstances.Include(i => i.SshHost).AsQueryable();
        if (!string.IsNullOrWhiteSpace(filter))
            query = query.Where(i => i.CustomerName.Contains(filter) || i.StackName.Contains(filter));

        var total = await query.CountAsync();
        var items = await query.OrderByDescending(i => i.CreatedAt)
            .Skip((page - 1) * pageSize).Take(pageSize).ToListAsync();
        return (items, total);
    }

    public async Task<XiboTestResult> TestXiboConnectionAsync(Guid id)
    {
        var instance = await _db.CmsInstances.FindAsync(id)
            ?? throw new KeyNotFoundException($"Instance {id} not found.");

        if (string.IsNullOrEmpty(instance.XiboUsername) || string.IsNullOrEmpty(instance.XiboPassword))
            return new XiboTestResult { IsValid = false, Message = "No Xibo credentials stored." };

        var url = $"http://localhost:{instance.HostHttpPort}";
        var result = await _xibo.TestConnectionAsync(url, instance.XiboUsername, instance.XiboPassword);
        instance.XiboApiTestStatus = result.IsValid ? XiboApiTestStatus.Success : XiboApiTestStatus.Failed;
        instance.XiboApiTestedAt = DateTime.UtcNow;
        await _db.SaveChangesAsync();
        return result;
    }

    private async Task EnsureSecretMetadata(string name, bool isGlobal, string? customerName)
    {
        var existing = await _db.SecretMetadata.FirstOrDefaultAsync(s => s.Name == name);
        if (existing == null)
        {
            _db.SecretMetadata.Add(new SecretMetadata
            {
                Name = name,
                IsGlobal = isGlobal,
                CustomerName = customerName
            });
        }
    }

    private static OperationLog StartOperation(OperationType type, string? userId)
        => new OperationLog { Operation = type, UserId = userId, Status = OperationStatus.Pending };

    private static string GenerateRandomPassword(int length)
    {
        const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*";
        return RandomNumberGenerator.GetString(chars, length);
    }
}