using System.Diagnostics;
using System.Security.Cryptography;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
using OTSSignsOrchestrator.Core.Configuration;
using OTSSignsOrchestrator.Core.Data;
using OTSSignsOrchestrator.Core.Models.DTOs;
using OTSSignsOrchestrator.Core.Models.Entities;
using static OTSSignsOrchestrator.Core.Configuration.AppConstants;

namespace OTSSignsOrchestrator.Core.Services;

/// <summary>
/// Orchestrator service for CMS instance lifecycle (Create, Update, Delete).
/// All instance data is live — nothing is persisted locally beyond the audit log.
/// New-instance flow:
///   1. Clone template repo to local cache
///   2. Generate MySQL password → create Docker Swarm secret (never persisted locally)
///   3. Create MySQL database + user on external MySQL server via SSH
///   4. Render combined compose YAML (no MySQL container, NFS volumes, Newt service)
///   5. Deploy stack via SSH
/// </summary>
public class InstanceService
{
    private readonly XiboContext _db;
    private readonly GitTemplateService _git;
    private readonly ComposeRenderService _compose;
    private readonly ComposeValidationService _validation;
    private readonly IDockerCliService _docker;
    private readonly IDockerSecretsService _secrets;
    private readonly XiboApiService _xibo;
    private readonly SettingsService _settings;
    private readonly DockerOptions _dockerOptions;
    private readonly ILogger<InstanceService> _logger;

    public InstanceService(
        XiboContext db,
        GitTemplateService git,
        ComposeRenderService compose,
        ComposeValidationService validation,
        IDockerCliService docker,
        IDockerSecretsService secrets,
        XiboApiService xibo,
        SettingsService settings,
        IOptions<DockerOptions> dockerOptions,
        ILogger<InstanceService> logger)
    {
        _db = db;
        _git = git;
        _compose = compose;
        _validation = validation;
        _docker = docker;
        _secrets = secrets;
        _xibo = xibo;
        _settings = settings;
        _dockerOptions = dockerOptions.Value;
        _logger = logger;
    }

    // ─────────────────────────────────────────────────────────────────────────
    // Helper: derive the 3-letter abbreviation from a stack name like "acm-cms-stack"
    // ─────────────────────────────────────────────────────────────────────────
    private static string ExtractAbbrev(string stackName)
        => stackName.EndsWith("-cms-stack")
            ? stackName[..^"-cms-stack".Length]
            : stackName.Split('-')[0];

    // ─────────────────────────────────────────────────────────────────────────
    // Create
    // ─────────────────────────────────────────────────────────────────────────

    /// <summary>
    /// Creates a new CMS instance:
    /// 1. Clone repo  2. Generate secrets  3. Create MySQL DB/user  4. Render compose  5. Deploy
    /// </summary>
    public async Task<DeploymentResultDto> CreateInstanceAsync(CreateInstanceDto dto, string? userId = null)
    {
        var sw = Stopwatch.StartNew();
        var abbrev = dto.CustomerAbbrev.Trim().ToLowerInvariant();
        var stackName = $"{abbrev}-cms-stack";
        var opLog = StartOperation(OperationType.Create, userId, stackName);

        try
        {
            _logger.LogInformation("Creating instance: abbrev={Abbrev}, customer={Customer}", abbrev, dto.CustomerName);

            // ── 1. Clone / refresh template repo ────────────────────────────
            var repoUrl = await _settings.GetAsync(SettingsService.GitRepoUrl);
            var repoPat = await _settings.GetAsync(SettingsService.GitRepoPat);

            if (string.IsNullOrWhiteSpace(repoUrl))
                throw new InvalidOperationException("Git template repository URL is not configured. Set it in Settings → Git Repo URL.");

            _logger.LogInformation("Fetching template repo: {RepoUrl}", repoUrl);
            var templateConfig = await _git.FetchAsync(repoUrl, repoPat);

            // ── 1b. Remove any stale stack that might hold references to old secrets ─
            _logger.LogInformation("Removing stale stack (if any): {StackName}", stackName);
            await _docker.RemoveStackAsync(stackName);
            await Task.Delay(2000);

            // ── 2. Generate MySQL credentials ──────────────────────────────
            var mysqlPassword = GenerateRandomPassword(32);
            var mySqlUserName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);

            var userSecretName = CustomerMysqlUserSecretName(abbrev);

            var (userOk, _) = await _secrets.EnsureSecretAsync(userSecretName, mySqlUserName, rotate: true);
            if (!userOk)
                throw new InvalidOperationException($"Failed to create/rotate Docker secret '{userSecretName}'.");
            _logger.LogInformation("Docker secret created/rotated: {SecretName}", userSecretName);

            // Global secrets: MySQL host + port (idempotent, shared across instances)
            var mySqlHostValue = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
            var mySqlPortValue = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
            await _secrets.EnsureSecretAsync(GlobalMysqlHostSecretName, mySqlHostValue);
            await _secrets.EnsureSecretAsync(GlobalMysqlPortSecretName, mySqlPortValue);
            _logger.LogInformation("Global MySQL secrets ensured: {Host}, {Port}", GlobalMysqlHostSecretName, GlobalMysqlPortSecretName);

            // ── 2b. Create MySQL database + user ─────────────────────────────
            _logger.LogInformation("Creating MySQL database and user for instance {Abbrev}", abbrev);
            var (mysqlOk, mysqlMsg) = await CreateMySqlDatabaseAsync(abbrev, mysqlPassword);
            if (!mysqlOk)
                throw new InvalidOperationException($"MySQL database/user setup failed: {mysqlMsg}");
            _logger.LogInformation("MySQL setup complete: {Message}", mysqlMsg);

            // ── 2c. Persist password (encrypted) for future redeploys ────────
            await _settings.SetAsync(
                SettingsService.InstanceMySqlPassword(abbrev), mysqlPassword,
                SettingsService.CatInstance, isSensitive: true);
            await _db.SaveChangesAsync();
            _logger.LogInformation("MySQL password stored in settings for instance {Abbrev}", abbrev);

            // ── 3. Read settings ────────────────────────────────────────────
            var mySqlHost    = mySqlHostValue;
            var mySqlPort    = mySqlPortValue;
            var mySqlDbName  = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
            var mySqlUser    = mySqlUserName;

            var cmsServerName = (await _settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "{abbrev}.ots-signs.com")).Replace("{abbrev}", abbrev);
            var themePath     = (await _settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme")).Replace("{abbrev}", abbrev);

            var smtpServer          = await _settings.GetAsync(SettingsService.SmtpServer, string.Empty);
            var smtpUsername        = await _settings.GetAsync(SettingsService.SmtpUsername, string.Empty);
            var smtpPassword        = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty);
            var smtpUseTls          = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES");
            var smtpUseStartTls     = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES");
            var smtpRewriteDomain   = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty);
            var smtpHostname        = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty);
            var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO");

            var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net");

            var nfsServer       = dto.NfsServer       ?? await _settings.GetAsync(SettingsService.NfsServer);
            var nfsExport       = dto.NfsExport       ?? await _settings.GetAsync(SettingsService.NfsExport);
            var nfsExportFolder = dto.NfsExportFolder ?? await _settings.GetAsync(SettingsService.NfsExportFolder);
            var nfsOptions      = dto.NfsExtraOptions ?? await _settings.GetAsync(SettingsService.NfsOptions, string.Empty);

            var cmsImage        = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3");
            var newtImage       = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt");
            var memcachedImage  = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine");
            var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart");

            var phpPostMaxSize       = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G");
            var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G");
            var phpMaxExecutionTime  = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600");

            // ── 4. Render compose YAML from template ────────────────────────
            var renderCtx = new RenderContext
            {
                CustomerName         = dto.CustomerName,
                CustomerAbbrev       = abbrev,
                StackName            = stackName,
                CmsServerName        = cmsServerName,
                HostHttpPort         = 80,
                CmsImage             = cmsImage,
                MemcachedImage       = memcachedImage,
                QuickChartImage      = quickChartImage,
                NewtImage            = newtImage,
                ThemeHostPath        = themePath,
                MySqlHost            = mySqlHost,
                MySqlPort            = mySqlPort,
                MySqlDatabase        = mySqlDbName,
                MySqlUser            = mySqlUser,
                MySqlPassword        = mysqlPassword,
                SmtpServer           = smtpServer,
                SmtpUsername         = smtpUsername,
                SmtpPassword         = smtpPassword,
                SmtpUseTls           = smtpUseTls,
                SmtpUseStartTls      = smtpUseStartTls,
                SmtpRewriteDomain    = smtpRewriteDomain,
                SmtpHostname         = smtpHostname,
                SmtpFromLineOverride = smtpFromLineOverride,
                PhpPostMaxSize       = phpPostMaxSize,
                PhpUploadMaxFilesize = phpUploadMaxFilesize,
                PhpMaxExecutionTime  = phpMaxExecutionTime,
                PangolinEndpoint     = pangolinEndpoint,
                NewtId               = dto.NewtId,
                NewtSecret           = dto.NewtSecret,
                NfsServer            = nfsServer,
                NfsExport            = nfsExport,
                NfsExportFolder      = nfsExportFolder,
                NfsExtraOptions      = nfsOptions,
            };

            _logger.LogInformation("NFS render values: server={NfsServer}, export={NfsExport}, folder={NfsExportFolder}",
                nfsServer, nfsExport, nfsExportFolder);

            var composeYaml = _compose.Render(templateConfig.Yaml, renderCtx);

            if (_dockerOptions.ValidateBeforeDeploy)
            {
                var validationResult = _validation.Validate(composeYaml, abbrev);
                if (!validationResult.IsValid)
                    throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}");
            }

            // ── 5. Ensure bind-mount directories exist on the remote host ───
            if (!string.IsNullOrWhiteSpace(themePath))
                await _docker.EnsureDirectoryAsync(themePath);

            // ── 5b. Ensure NFS export folders exist ─────────────────────────
            if (!string.IsNullOrWhiteSpace(nfsServer) && !string.IsNullOrWhiteSpace(nfsExport))
            {
                var nfsFolders = ComposeRenderService.ExtractNfsDeviceFolders(composeYaml, nfsExport, nfsExportFolder);
                _logger.LogInformation("Ensuring NFS export folders exist on {Server}:{Export} — {Folders}",
                    nfsServer, nfsExport, string.Join(", ", nfsFolders));
                var (nfsFoldersOk, nfsError) = await _docker.EnsureNfsFoldersWithErrorAsync(nfsServer, nfsExport, nfsFolders, nfsExportFolder);
                if (!nfsFoldersOk)
                    throw new InvalidOperationException(
                        $"Failed to create NFS volume directories on {nfsServer}:{nfsExport}: {nfsError}\n"
                        + "Common causes: (1) SSH user needs passwordless sudo (NOPASSWD) for mount/umount/mkdir, "
                        + "(2) NFS export has root_squash enabled — set 'No mapping' / no_root_squash on the NFS server.");
            }

            // ── 6. Remove stale NFS volumes ─────────────────────────────────
            _logger.LogInformation("Removing stale NFS volumes for stack {StackName}", stackName);
            await _docker.RemoveStackVolumesAsync(stackName);

            // ── 7. Deploy stack ─────────────────────────────────────────────
            var deployResult = await _docker.DeployStackAsync(stackName, composeYaml);
            if (!deployResult.Success)
                throw new InvalidOperationException($"Stack deployment failed: {deployResult.ErrorMessage}");

            mysqlPassword = string.Empty;

            sw.Stop();
            opLog.Status      = OperationStatus.Success;
            opLog.Message     = $"Instance deployed: {stackName}";
            opLog.DurationMs  = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();

            _logger.LogInformation("Instance created: {StackName} | duration={DurationMs}ms", stackName, sw.ElapsedMilliseconds);

            deployResult.ServiceCount = 4;
            deployResult.Message      = "Instance deployed successfully.";
            return deployResult;
        }
        catch (Exception ex)
        {
            sw.Stop();
            opLog.Status     = OperationStatus.Failure;
            opLog.Message    = $"Create failed: {ex.Message}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();
            _logger.LogError(ex, "Instance create failed: {StackName}", stackName);
            throw;
        }
    }

    // ─────────────────────────────────────────────────────────────────────────
    // Update / redeploy
    // ─────────────────────────────────────────────────────────────────────────

    /// <summary>
    /// Redeploys an existing stack. All render parameters come from <paramref name="dto"/>
    /// (populated from live service inspection or user input), falling back to global settings
    /// where values are null.
    /// </summary>
    public async Task<DeploymentResultDto> UpdateInstanceAsync(string stackName, UpdateInstanceDto dto, string? userId = null)
    {
        var sw    = Stopwatch.StartNew();
        var opLog = StartOperation(OperationType.Update, userId, stackName);

        try
        {
            var abbrev       = dto.CustomerAbbrev?.Trim().ToLowerInvariant() ?? ExtractAbbrev(stackName);
            var customerName = dto.CustomerName ?? abbrev;

            _logger.LogInformation("Updating instance: {StackName} (abbrev={Abbrev})", stackName, abbrev);

            var mySqlHost   = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
            var mySqlPort   = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
            var mySqlDbName = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
            var mySqlUser   = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);

            // Retrieve the stored MySQL password (encrypted in settings since creation)
            var mySqlPassword = await _settings.GetAsync(SettingsService.InstanceMySqlPassword(abbrev));
            if (string.IsNullOrEmpty(mySqlPassword))
                throw new InvalidOperationException(
                    $"No stored MySQL password found for instance '{abbrev}'. " +
                    "The instance may have been created before password persistence was added. " +
                    "Use 'Rotate Password' to generate and store a new one.");

            // Ensure MySQL Docker secrets exist (idempotent)
            var userSecretName = CustomerMysqlUserSecretName(abbrev);
            await _secrets.EnsureSecretAsync(userSecretName, mySqlUser);
            await _secrets.EnsureSecretAsync(GlobalMysqlHostSecretName, mySqlHost);
            await _secrets.EnsureSecretAsync(GlobalMysqlPortSecretName, mySqlPort);

            var smtpServer           = dto.SmtpServer   ?? await _settings.GetAsync(SettingsService.SmtpServer, string.Empty);
            var smtpUsername         = dto.SmtpUsername ?? await _settings.GetAsync(SettingsService.SmtpUsername, string.Empty);
            var smtpPassword         = await _settings.GetAsync(SettingsService.SmtpPassword, string.Empty);
            var smtpUseTls           = await _settings.GetAsync(SettingsService.SmtpUseTls, "YES");
            var smtpUseStartTls      = await _settings.GetAsync(SettingsService.SmtpUseStartTls, "YES");
            var smtpRewriteDomain    = await _settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty);
            var smtpHostname         = await _settings.GetAsync(SettingsService.SmtpHostname, string.Empty);
            var smtpFromLineOverride = await _settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO");

            var pangolinEndpoint = await _settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net");

            var cmsServerName = dto.CmsServerName
                ?? (await _settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "{abbrev}.ots-signs.com")).Replace("{abbrev}", abbrev);
            var hostHttpPort = dto.HostHttpPort ?? 80;
            var themePath    = dto.ThemeHostPath
                ?? (await _settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme")).Replace("{abbrev}", abbrev);

            var nfsServer       = dto.NfsServer       ?? await _settings.GetAsync(SettingsService.NfsServer);
            var nfsExport       = dto.NfsExport       ?? await _settings.GetAsync(SettingsService.NfsExport);
            var nfsExportFolder = dto.NfsExportFolder ?? await _settings.GetAsync(SettingsService.NfsExportFolder);
            var nfsOptions      = dto.NfsExtraOptions ?? await _settings.GetAsync(SettingsService.NfsOptions, string.Empty);

            var cmsImage        = await _settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3");
            var newtImage       = await _settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt");
            var memcachedImage  = await _settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine");
            var quickChartImage = await _settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart");

            var phpPostMaxSize       = await _settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G");
            var phpUploadMaxFilesize = await _settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G");
            var phpMaxExecutionTime  = await _settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600");

            var repoUrl = dto.TemplateRepoUrl ?? await _settings.GetAsync(SettingsService.GitRepoUrl);
            var repoPat = dto.TemplateRepoPat ?? await _settings.GetAsync(SettingsService.GitRepoPat);

            if (string.IsNullOrWhiteSpace(repoUrl))
                throw new InvalidOperationException("Git template repository URL is not configured. Set it in Settings → Git Repo URL.");

            var templateConfig = await _git.FetchAsync(repoUrl, repoPat);

            var renderCtx = new RenderContext
            {
                CustomerName         = customerName,
                CustomerAbbrev       = abbrev,
                StackName            = stackName,
                CmsServerName        = cmsServerName,
                HostHttpPort         = hostHttpPort,
                CmsImage             = cmsImage,
                MemcachedImage       = memcachedImage,
                QuickChartImage      = quickChartImage,
                NewtImage            = newtImage,
                ThemeHostPath        = themePath,
                MySqlHost            = mySqlHost,
                MySqlPort            = mySqlPort,
                MySqlDatabase        = mySqlDbName,
                MySqlUser            = mySqlUser,
                MySqlPassword        = mySqlPassword,
                SmtpServer           = smtpServer,
                SmtpUsername         = smtpUsername,
                SmtpPassword         = smtpPassword,
                SmtpUseTls           = smtpUseTls,
                SmtpUseStartTls      = smtpUseStartTls,
                SmtpRewriteDomain    = smtpRewriteDomain,
                SmtpHostname         = smtpHostname,
                SmtpFromLineOverride = smtpFromLineOverride,
                PhpPostMaxSize       = phpPostMaxSize,
                PhpUploadMaxFilesize = phpUploadMaxFilesize,
                PhpMaxExecutionTime  = phpMaxExecutionTime,
                PangolinEndpoint     = pangolinEndpoint,
                NewtId               = dto.NewtId,
                NewtSecret           = dto.NewtSecret,
                NfsServer            = nfsServer,
                NfsExport            = nfsExport,
                NfsExportFolder      = nfsExportFolder,
                NfsExtraOptions      = nfsOptions,
            };

            _logger.LogInformation("NFS render values (update): server={NfsServer}, export={NfsExport}, folder={NfsExportFolder}",
                nfsServer, nfsExport, nfsExportFolder);

            var composeYaml = _compose.Render(templateConfig.Yaml, renderCtx);

            if (_dockerOptions.ValidateBeforeDeploy)
            {
                var validationResult = _validation.Validate(composeYaml, abbrev);
                if (!validationResult.IsValid)
                    throw new InvalidOperationException($"Compose validation failed: {string.Join("; ", validationResult.Errors)}");
            }

            if (!string.IsNullOrWhiteSpace(themePath))
                await _docker.EnsureDirectoryAsync(themePath);

            if (!string.IsNullOrWhiteSpace(nfsServer) && !string.IsNullOrWhiteSpace(nfsExport))
            {
                var nfsFolders = ComposeRenderService.ExtractNfsDeviceFolders(composeYaml, nfsExport, nfsExportFolder);
                _logger.LogInformation("Ensuring NFS export folders on {Server}:{Export} — {Folders}",
                    nfsServer, nfsExport, string.Join(", ", nfsFolders));
                var (nfsFoldersOk, nfsError) = await _docker.EnsureNfsFoldersWithErrorAsync(nfsServer, nfsExport, nfsFolders, nfsExportFolder);
                if (!nfsFoldersOk)
                    throw new InvalidOperationException(
                        $"Failed to create NFS volume directories on {nfsServer}:{nfsExport}: {nfsError}\n"
                        + "Common causes: (1) SSH user needs passwordless sudo (NOPASSWD) for mount/umount/mkdir, "
                        + "(2) NFS export has root_squash enabled — set 'No mapping' / no_root_squash on the NFS server.");
            }

            _logger.LogInformation("Removing stale NFS volumes for stack {StackName}", stackName);
            await _docker.RemoveStackVolumesAsync(stackName);

            var deployResult = await _docker.DeployStackAsync(stackName, composeYaml, resolveImage: true);
            if (!deployResult.Success)
                throw new InvalidOperationException($"Stack redeploy failed: {deployResult.ErrorMessage}");

            sw.Stop();
            opLog.Status     = OperationStatus.Success;
            opLog.Message    = $"Instance updated: {stackName}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();

            deployResult.ServiceCount = 4;
            deployResult.Message      = "Instance updated and redeployed.";
            return deployResult;
        }
        catch (Exception ex)
        {
            sw.Stop();
            opLog.Status     = OperationStatus.Failure;
            opLog.Message    = $"Update failed: {ex.Message}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();
            _logger.LogError(ex, "Instance update failed: {StackName}", stackName);
            throw;
        }
    }

    // ─────────────────────────────────────────────────────────────────────────
    // Delete
    // ─────────────────────────────────────────────────────────────────────────

    public async Task<DeploymentResultDto> DeleteInstanceAsync(
        string stackName,
        string customerAbbrev,
        bool retainSecrets = false,
        bool clearXiboCreds = true,
        string? userId = null)
    {
        var sw    = Stopwatch.StartNew();
        var opLog = StartOperation(OperationType.Delete, userId, stackName);

        try
        {
            var abbrev = customerAbbrev.Trim().ToLowerInvariant();

            _logger.LogInformation("Deleting instance: {StackName} (abbrev={Abbrev}) retainSecrets={RetainSecrets}",
                stackName, abbrev, retainSecrets);

            var result = await _docker.RemoveStackAsync(stackName);

            if (!retainSecrets)
            {
                _logger.LogInformation("Dropping MySQL database and user for instance {StackName}", stackName);
                var (dropOk, dropMsg) = await DropMySqlDatabaseAsync(abbrev);
                if (!dropOk)
                    _logger.LogWarning("MySQL cleanup incomplete for {StackName}: {Message}", stackName, dropMsg);
                else
                    _logger.LogInformation("MySQL cleanup complete: {Message}", dropMsg);

                foreach (var secretName in AllCustomerMysqlSecretNames(abbrev))
                    await _secrets.DeleteSecretAsync(secretName);

                // Remove the stored password from local settings
                await _settings.SetAsync(
                    SettingsService.InstanceMySqlPassword(abbrev), null,
                    SettingsService.CatInstance, isSensitive: false);
                await _db.SaveChangesAsync();
            }

            sw.Stop();
            opLog.Status     = OperationStatus.Success;
            opLog.Message    = $"Instance deleted: {stackName}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();

            result.Message = "Instance deleted.";
            return result;
        }
        catch (Exception ex)
        {
            sw.Stop();
            opLog.Status     = OperationStatus.Failure;
            opLog.Message    = $"Delete failed: {ex.Message}";
            opLog.DurationMs = sw.ElapsedMilliseconds;
            _db.OperationLogs.Add(opLog);
            await _db.SaveChangesAsync();
            _logger.LogError(ex, "Instance delete failed: {StackName}", stackName);
            throw;
        }
    }

    // ─────────────────────────────────────────────────────────────────────────
    // MySQL password rotation
    // ─────────────────────────────────────────────────────────────────────────

    /// <summary>
    /// Rotates the MySQL password for an instance: generates a new password, updates the
    /// Docker Swarm secret, updates the MySQL user via ALTER USER, then redeploys the stack.
    /// </summary>
    public async Task<(bool Success, string Message)> RotateMySqlPasswordAsync(string stackName, string? userId = null)
    {
        var abbrev      = ExtractAbbrev(stackName);
        var newPassword = GenerateRandomPassword(32);

        _logger.LogInformation("Rotating MySQL password for instance {StackName}", stackName);

        // ── Step 1: Update MySQL password via SSH tunnel ─────────────────────
        var mySqlHost          = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
        var mySqlPort          = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
        var mySqlAdminUser     = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
        var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
        var mySqlUser          = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);

        if (!int.TryParse(mySqlPort, out var port)) port = 3306;

        var (mysqlOk, mysqlErr) = await _docker.AlterMySqlUserPasswordAsync(
            mySqlHost, port, mySqlAdminUser, mySqlAdminPassword, mySqlUser, newPassword);

        if (!mysqlOk)
            return (false, $"MySQL ALTER USER failed (no service disruption): {mysqlErr}");

        _logger.LogInformation("MySQL password updated for user {User}", mySqlUser);

        // ── Step 2: Persist new password in settings (encrypted) ─────────────
        await _settings.SetAsync(
            SettingsService.InstanceMySqlPassword(abbrev), newPassword,
            SettingsService.CatInstance, isSensitive: true);
        await _db.SaveChangesAsync();
        _logger.LogInformation("New MySQL password stored in settings for instance {Abbrev}", abbrev);

        // ── Step 3: Force-update the web service to pick up new compose env ──
        // Redeploy the stack so the MYSQL_PASSWORD env var gets the new value.
        var webService = $"{stackName}_{abbrev}-web";
        if (!await _docker.ForceUpdateServiceAsync(webService))
        {
            _logger.LogWarning(
                "Force-update of service {Service} failed. MySQL has the new password but " +
                "the container is still running with the old value. A manual redeploy may be needed.",
                webService);
            return (true, $"MySQL password rotated for '{stackName}' but service force-update failed. Redeploy to apply.");
        }

        _logger.LogInformation("Service {Service} force-updated with new password", webService);
        return (true, $"MySQL password rotated successfully for '{stackName}'.");
    }

    // ─────────────────────────────────────────────────────────────────────────
    // MySQL database helpers
    // ─────────────────────────────────────────────────────────────────────────

    /// <summary>
    /// Creates MySQL database and user on the external MySQL server via SSH tunnel.
    /// </summary>
    public async Task<(bool Success, string Message)> CreateMySqlDatabaseAsync(
        string abbrev,
        string mysqlPassword)
    {
        var mySqlHost          = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
        var mySqlPort          = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
        var mySqlAdminUser     = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
        var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);

        var dbName   = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
        var userName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);

        _logger.LogInformation("Creating MySQL database {Db} and user {User} via SSH tunnel", dbName, userName);

        if (!int.TryParse(mySqlPort, out var port))
            port = 3306;

        try
        {
            var (connection, tunnel) = await _docker.OpenMySqlConnectionAsync(
                mySqlHost, port, mySqlAdminUser, mySqlAdminPassword);
            await using var _ = connection;
            using var __ = tunnel;

            var escapedDb   = dbName.Replace("`", "``");
            var escapedUser = userName.Replace("'", "''");

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = $"CREATE DATABASE IF NOT EXISTS `{escapedDb}`";
                await cmd.ExecuteNonQueryAsync();
            }

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = $"CREATE USER IF NOT EXISTS '{escapedUser}'@'%' IDENTIFIED BY @pwd";
                cmd.Parameters.AddWithValue("@pwd", mysqlPassword);
                await cmd.ExecuteNonQueryAsync();
            }

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = $"ALTER USER '{escapedUser}'@'%' IDENTIFIED BY @pwd";
                cmd.Parameters.AddWithValue("@pwd", mysqlPassword);
                await cmd.ExecuteNonQueryAsync();
            }

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = $"GRANT ALL PRIVILEGES ON `{escapedDb}`.* TO '{escapedUser}'@'%'";
                await cmd.ExecuteNonQueryAsync();
            }

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = "FLUSH PRIVILEGES";
                await cmd.ExecuteNonQueryAsync();
            }

            // Verify the new user's credentials by opening a connection as that user
            try
            {
                var parsed = new MySqlConnectionStringBuilder(connection.ConnectionString);
                var localPort = (int)parsed.Port;

                var testCsb = new MySqlConnectionStringBuilder
                {
                    Server = "127.0.0.1",
                    Port = (uint)localPort,
                    UserID = userName,
                    Password = mysqlPassword,
                    ConnectionTimeout = 5,
                    SslMode = MySqlSslMode.Disabled,
                };

                await using var testConn = new MySqlConnection(testCsb.ConnectionString);
                await testConn.OpenAsync();
                await testConn.CloseAsync();
            }
            catch (MySqlException ex)
            {
                _logger.LogError(ex, "Verification login failed for new MySQL user {User}", userName);
                return (false, $"Verification login failed for user '{userName}': {ex.Message}");
            }

            _logger.LogInformation("MySQL database {Db} and user {User} created and verified successfully", dbName, userName);
            return (true, $"Database '{dbName}' and user '{userName}' created.");
        }
        catch (MySqlException ex)
        {
            _logger.LogError(ex, "MySQL setup failed for database {Db}", dbName);
            return (false, $"MySQL setup failed: {ex.Message}");
        }
    }

    /// <summary>
    /// Drops the MySQL database and user for a given instance abbreviation.
    /// </summary>
    public async Task<(bool Success, string Message)> DropMySqlDatabaseAsync(string abbrev)
    {
        var mySqlHost          = await _settings.GetAsync(SettingsService.MySqlHost, "localhost");
        var mySqlPort          = await _settings.GetAsync(SettingsService.MySqlPort, "3306");
        var mySqlAdminUser     = await _settings.GetAsync(SettingsService.MySqlAdminUser, "root");
        var mySqlAdminPassword = await _settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);

        var dbName   = (await _settings.GetAsync(SettingsService.DefaultMySqlDbTemplate, "{abbrev}_cms_db")).Replace("{abbrev}", abbrev);
        var userName = (await _settings.GetAsync(SettingsService.DefaultMySqlUserTemplate, "{abbrev}_cms_user")).Replace("{abbrev}", abbrev);

        _logger.LogInformation("Dropping MySQL database {Db} and user {User} via SSH tunnel", dbName, userName);

        if (!int.TryParse(mySqlPort, out var port))
            port = 3306;

        try
        {
            var (connection, tunnel) = await _docker.OpenMySqlConnectionAsync(
                mySqlHost, port, mySqlAdminUser, mySqlAdminPassword);
            await using var _ = connection;
            using var __ = tunnel;

            var escapedDb   = dbName.Replace("`", "``");
            var escapedUser = userName.Replace("'", "''");

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = $"DROP DATABASE IF EXISTS `{escapedDb}`";
                await cmd.ExecuteNonQueryAsync();
            }

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = $"DROP USER IF EXISTS '{escapedUser}'@'%'";
                await cmd.ExecuteNonQueryAsync();
            }

            await using (var cmd = connection.CreateCommand())
            {
                cmd.CommandText = "FLUSH PRIVILEGES";
                await cmd.ExecuteNonQueryAsync();
            }

            _logger.LogInformation("MySQL database {Db} and user {User} dropped successfully", dbName, userName);
            return (true, $"Database '{dbName}' and user '{userName}' dropped.");
        }
        catch (MySqlException ex)
        {
            _logger.LogError(ex, "MySQL drop failed for database {Db}", dbName);
            return (false, $"MySQL drop failed: {ex.Message}");
        }
    }

    // ─────────────────────────────────────────────────────────────────────────
    // Private helpers
    // ─────────────────────────────────────────────────────────────────────────

    private static OperationLog StartOperation(OperationType type, string? userId, string? stackName = null)
        => new OperationLog { Operation = type, UserId = userId, Status = OperationStatus.Pending, StackName = stackName };

    private static string GenerateRandomPassword(int length)
    {
        const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
        return RandomNumberGenerator.GetString(chars, length);
    }
}


/// <summary>
/// Orchestrator service for CMS instance lifecycle (Create, Update, Delete, List, Inspect).