Add WAL file for database and log instance deployment failures

OTSSignsOrchestrator.Core/Services/ComposeRenderService.cs

@@ -1,3 +1,4 @@
+using System.Text.RegularExpressions;
 using Microsoft.Extensions.Logging;
 
 namespace OTSSignsOrchestrator.Core.Services;
@@ -28,7 +29,8 @@ public class ComposeRenderService
         if (string.IsNullOrWhiteSpace(templateYaml))
             throw new ArgumentException("Template YAML is empty. Ensure template.yml exists in the configured git repository.");
 
-        var cifsOpts = BuildCifsOpts(ctx);
+        var nfsOpts = BuildNfsOpts(ctx);
+        var nfsDevicePrefix = BuildNfsDevicePrefix(ctx);
 
         return templateYaml
             .Replace("{{ABBREV}}",                    ctx.CustomerAbbrev)
@@ -59,40 +61,87 @@ public class ComposeRenderService
             .Replace("{{PANGOLIN_ENDPOINT}}",         ctx.PangolinEndpoint)
             .Replace("{{NEWT_ID}}",                   ctx.NewtId ?? "CONFIGURE_ME")
             .Replace("{{NEWT_SECRET}}",               ctx.NewtSecret ?? "CONFIGURE_ME")
-            .Replace("{{CIFS_SERVER}}",              (ctx.CifsServer ?? string.Empty).TrimEnd('/'))
-            .Replace("{{CIFS_SHARE_NAME}}",          BuildSharePath(ctx.CifsShareName, ctx.CifsShareFolder))
-            // Legacy token — was a path component (e.g. "/sharename"), so templates concatenate
-            // it directly after the server: //{{CIFS_SERVER}}{{CIFS_SHARE_BASE_PATH}}/...
-            // We must keep the leading "/" to produce a valid device path.
-            .Replace("{{CIFS_SHARE_BASE_PATH}}",     "/" + BuildSharePath(ctx.CifsShareName, ctx.CifsShareFolder))
-            .Replace("{{CIFS_USERNAME}}",             ctx.CifsUsername ?? string.Empty)
-            .Replace("{{CIFS_PASSWORD}}",             ctx.CifsPassword ?? string.Empty)
-            .Replace("{{CIFS_OPTS}}",                 cifsOpts);
+            .Replace("{{NFS_DEVICE_PREFIX}}",         nfsDevicePrefix)
+            .Replace("{{NFS_OPTS}}",                  nfsOpts)
+            // ── Legacy CIFS token compatibility ─────────────────────────────
+            // External git template repos may still contain old CIFS tokens.
+            // Map them to NFS equivalents so those templates render correctly.
+            .Replace("{{CIFS_SERVER}}",              ctx.NfsServer ?? string.Empty)
+            .Replace("{{CIFS_SHARE_NAME}}",          BuildLegacySharePath(ctx))
+            .Replace("{{CIFS_SHARE_BASE_PATH}}",     "/" + BuildLegacySharePath(ctx))
+            .Replace("{{CIFS_USERNAME}}",            string.Empty)
+            .Replace("{{CIFS_PASSWORD}}",            string.Empty)
+            .Replace("{{CIFS_OPTS}}",                nfsOpts);
     }
 
-    private static string BuildCifsOpts(RenderContext ctx)
+    /// <summary>
+    /// Builds a legacy-compatible share path from NFS export + folder for old CIFS templates.
+    /// Maps NFS export/folder to the path that was previously the CIFS share name/folder.
+    /// </summary>
+    private static string BuildLegacySharePath(RenderContext ctx)
     {
-        if (string.IsNullOrWhiteSpace(ctx.CifsServer))
+        var export = (ctx.NfsExport ?? string.Empty).Trim('/');
+        var folder = (ctx.NfsExportFolder ?? string.Empty).Trim('/');
+        return string.IsNullOrEmpty(folder) ? export : $"{export}/{folder}";
+    }
+
+    /// <summary>
+    /// Builds the NFS mount options string for Docker volume driver_opts.
+    /// Format: "addr=&lt;server&gt;,nfsvers=4,proto=tcp[,extraOptions]".
+    /// </summary>
+    private static string BuildNfsOpts(RenderContext ctx)
+    {
+        if (string.IsNullOrWhiteSpace(ctx.NfsServer))
             return string.Empty;
 
-        // vers=3.0 is required by most modern SMB servers (e.g. Hetzner Storage Box).
-        // Without it, mount.cifs may negotiate SMBv1/2.1 which gets rejected as "permission denied".
-        var opts = $"addr={ctx.CifsServer},username={ctx.CifsUsername},password={ctx.CifsPassword},vers=3.0";
-        if (!string.IsNullOrWhiteSpace(ctx.CifsExtraOptions))
-            opts += $",{ctx.CifsExtraOptions}";
+        var opts = $"addr={ctx.NfsServer},nfsvers=4,proto=tcp";
+        if (!string.IsNullOrWhiteSpace(ctx.NfsExtraOptions))
+            opts += $",{ctx.NfsExtraOptions}";
         return opts;
     }
 
     /// <summary>
-    /// Combines share name and optional subfolder into a single path segment.
-    /// e.g. ("u548897-sub1", "ots_cms") → "u548897-sub1/ots_cms"
-    ///      ("u548897-sub1", null)       → "u548897-sub1"
+    /// Builds the NFS device prefix used in volume definitions.
+    /// Format: ":/export[/subfolder]"  (the colon is part of the device path for NFS).
+    /// e.g. ":/srv/nfs/ots_cms" or ":/srv/nfs".
     /// </summary>
-    private static string BuildSharePath(string? shareName, string? shareFolder)
+    private static string BuildNfsDevicePrefix(RenderContext ctx)
     {
-        var name = (shareName ?? string.Empty).Trim('/');
-        var folder = (shareFolder ?? string.Empty).Trim('/');
-        return string.IsNullOrEmpty(folder) ? name : $"{name}/{folder}";
+        var export = (ctx.NfsExport ?? string.Empty).Trim('/');
+        var folder = (ctx.NfsExportFolder ?? string.Empty).Trim('/');
+        var path = string.IsNullOrEmpty(folder) ? export : $"{export}/{folder}";
+        return $":/{path}";
+    }
+
+    /// <summary>
+    /// Extracts NFS volume device paths from rendered compose YAML, then strips the
+    /// NFS export prefix to return just the relative folder paths that need to exist
+    /// on the NFS server.  Works regardless of the template's naming convention
+    /// (hierarchical <c>ots/cms-custom</c> vs flat <c>ots-cms-custom</c>).
+    /// </summary>
+    public static List<string> ExtractNfsDeviceFolders(string renderedYaml, string nfsExport, string? nfsExportFolder = null)
+    {
+        // NFS device lines look like:  device: ":/mnt/Export/folder/ots-cms-custom"
+        // The colon prefix is the NFS device convention.
+        var matches = Regex.Matches(renderedYaml, @"device:\s*""?:(/[^""]+)""?", RegexOptions.IgnoreCase);
+        var export = (nfsExport ?? string.Empty).Trim('/');
+        var subFolder = (nfsExportFolder ?? string.Empty).Trim('/');
+        var prefix = string.IsNullOrEmpty(subFolder) ? $"/{export}" : $"/{export}/{subFolder}";
+
+        var folders = new List<string>();
+        foreach (Match match in matches)
+        {
+            var devicePath = match.Groups[1].Value;  // e.g. /mnt/DS-SwarmVolumes/Volumes/ots-cms-custom
+            if (devicePath.StartsWith(prefix + "/"))
+            {
+                // Strip the export prefix to get the relative folder path
+                var relative = devicePath[(prefix.Length + 1)..]; // e.g. ots-cms-custom or ots/cms-custom
+                if (!string.IsNullOrEmpty(relative))
+                    folders.Add(relative);
+            }
+        }
+
+        return folders;
     }
 
     /// <summary>
@@ -134,6 +183,9 @@ public class ComposeRenderService
               CMS_PHP_MAX_EXECUTION_TIME: "{{PHP_MAX_EXECUTION_TIME}}"
             secrets:
               - {{ABBREV}}-cms-db-password
+              - {{ABBREV}}-cms-db-user
+              - global_mysql_host
+              - global_mysql_port
             volumes:
               - {{ABBREV}}-cms-custom:/var/www/cms/custom
               - {{ABBREV}}-cms-backup:/var/www/backup
@@ -199,37 +251,43 @@ public class ComposeRenderService
           {{ABBREV}}-cms-custom:
             driver: local
             driver_opts:
-              type: cifs
-              device: //{{CIFS_SERVER}}/{{CIFS_SHARE_NAME}}/{{ABBREV}}-cms-custom
-              o: {{CIFS_OPTS}}
+              type: nfs
+              device: "{{NFS_DEVICE_PREFIX}}/{{ABBREV}}/cms-custom"
+              o: "{{NFS_OPTS}}"
           {{ABBREV}}-cms-backup:
             driver: local
             driver_opts:
-              type: cifs
-              device: //{{CIFS_SERVER}}/{{CIFS_SHARE_NAME}}/{{ABBREV}}-cms-backup
-              o: {{CIFS_OPTS}}
+              type: nfs
+              device: "{{NFS_DEVICE_PREFIX}}/{{ABBREV}}/cms-backup"
+              o: "{{NFS_OPTS}}"
           {{ABBREV}}-cms-library:
             driver: local
             driver_opts:
-              type: cifs
-              device: //{{CIFS_SERVER}}/{{CIFS_SHARE_NAME}}/{{ABBREV}}-cms-library
-              o: {{CIFS_OPTS}}
+              type: nfs
+              device: "{{NFS_DEVICE_PREFIX}}/{{ABBREV}}/cms-library"
+              o: "{{NFS_OPTS}}"
           {{ABBREV}}-cms-userscripts:
             driver: local
             driver_opts:
-              type: cifs
-              device: //{{CIFS_SERVER}}/{{CIFS_SHARE_NAME}}/{{ABBREV}}-cms-userscripts
-              o: {{CIFS_OPTS}}
+              type: nfs
+              device: "{{NFS_DEVICE_PREFIX}}/{{ABBREV}}/cms-userscripts"
+              o: "{{NFS_OPTS}}"
           {{ABBREV}}-cms-ca-certs:
             driver: local
             driver_opts:
-              type: cifs
-              device: //{{CIFS_SERVER}}/{{CIFS_SHARE_NAME}}/{{ABBREV}}-cms-ca-certs
-              o: {{CIFS_OPTS}}
+              type: nfs
+              device: "{{NFS_DEVICE_PREFIX}}/{{ABBREV}}/cms-ca-certs"
+              o: "{{NFS_OPTS}}"
 
         secrets:
           {{ABBREV}}-cms-db-password:
             external: true
+          {{ABBREV}}-cms-db-user:
+            external: true
+          global_mysql_host:
+            external: true
+          global_mysql_port:
+            external: true
         """;
 }
 
@@ -277,12 +335,12 @@ public class RenderContext
     public string? NewtId { get; set; }
     public string? NewtSecret { get; set; }
 
-    // CIFS volume settings
-    public string? CifsServer { get; set; }
-    public string? CifsShareName { get; set; }
-    /// <summary>Optional subfolder within the share (e.g. "ots_cms"). Empty/null = share root.</summary>
-    public string? CifsShareFolder { get; set; }
-    public string? CifsUsername { get; set; }
-    public string? CifsPassword { get; set; }
-    public string? CifsExtraOptions { get; set; }
+    // NFS volume settings
+    public string? NfsServer { get; set; }
+    /// <summary>NFS export path on the server (e.g. "/srv/nfs" or "/export/data").</summary>
+    public string? NfsExport { get; set; }
+    /// <summary>Optional subfolder within the export (e.g. "ots_cms"). Empty/null = export root.</summary>
+    public string? NfsExportFolder { get; set; }
+    /// <summary>Additional NFS mount options appended after the defaults (nfsvers=4,proto=tcp).</summary>
+    public string? NfsExtraOptions { get; set; }
 }

OTSSignsOrchestrator.Core/Services/ComposeValidationService.cs

@@ -1,4 +1,5 @@
 using Microsoft.Extensions.Logging;
+using OTSSignsOrchestrator.Core.Configuration;
 using YamlDotNet.RepresentationModel;
 
 namespace OTSSignsOrchestrator.Core.Services;
@@ -91,6 +92,11 @@ public class ComposeValidationService
 
         if (HasKey(root, "secrets") && root.Children[new YamlScalarNode("secrets")] is YamlMappingNode secretsNode)
         {
+            var presentSecrets = secretsNode.Children.Keys
+                .OfType<YamlScalarNode>()
+                .Select(k => k.Value!)
+                .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
             foreach (var (key, value) in secretsNode.Children)
             {
                 if (value is YamlMappingNode secretNode)
@@ -99,6 +105,23 @@ public class ComposeValidationService
                         warnings.Add($"Secret '{(key as YamlScalarNode)?.Value}' is not external.");
                 }
             }
+
+            // Validate that all required MySQL secrets are declared
+            if (!string.IsNullOrEmpty(customerAbbrev))
+            {
+                var requiredSecrets = new[]
+                {
+                    AppConstants.CustomerMysqlPasswordSecretName(customerAbbrev),
+                    AppConstants.CustomerMysqlUserSecretName(customerAbbrev),
+                    AppConstants.GlobalMysqlHostSecretName,
+                    AppConstants.GlobalMysqlPortSecretName,
+                };
+                foreach (var required in requiredSecrets)
+                {
+                    if (!presentSecrets.Contains(required))
+                        errors.Add($"Missing required secret: '{required}'.");
+                }
+            }
         }
 
         return new ValidationResult { Errors = errors, Warnings = warnings };

OTSSignsOrchestrator.Core/Services/IDockerCliService.cs

@@ -1,3 +1,4 @@
+using MySqlConnector;
 using OTSSignsOrchestrator.Core.Models.DTOs;
 
 namespace OTSSignsOrchestrator.Core.Services;
@@ -17,25 +18,75 @@ public interface IDockerCliService
     Task<bool> EnsureDirectoryAsync(string path);
 
     /// <summary>
-    /// Ensures the required folders exist on an SMB/CIFS share, creating any that are missing.
-    /// If <paramref name="cifsShareFolder"/> is non-empty, creates it first as a subfolder of the share,
+    /// Ensures the required folders exist on an NFS export, creating any that are missing.
+    /// If <paramref name="nfsExportFolder"/> is non-empty, creates it first as a subfolder of the export,
     /// then creates the volume folders inside it.
-    /// Uses smbclient on the remote host to interact with the share without requiring a mount.
+    /// Temporarily mounts the NFS export on the Docker host to create the directories.
     /// </summary>
-    Task<bool> EnsureSmbFoldersAsync(
-        string cifsServer,
-        string cifsShareName,
-        string cifsUsername,
-        string cifsPassword,
+    Task<bool> EnsureNfsFoldersAsync(
+        string nfsServer,
+        string nfsExport,
         IEnumerable<string> folderNames,
-        string? cifsShareFolder = null);
+        string? nfsExportFolder = null);
+
+    /// <summary>
+    /// Same as <see cref="EnsureNfsFoldersAsync"/> but returns the error message on failure
+    /// so callers can surface actionable diagnostics.
+    /// </summary>
+    Task<(bool Success, string? Error)> EnsureNfsFoldersWithErrorAsync(
+        string nfsServer,
+        string nfsExport,
+        IEnumerable<string> folderNames,
+        string? nfsExportFolder = null);
 
     /// <summary>
     /// Removes all Docker volumes whose names start with <paramref name="stackName"/>_.
     /// Volumes currently in use by running containers will be skipped.
-    /// Safe for CIFS volumes since data lives on the remote share, not in the local volume.
+    /// Safe for NFS volumes since data lives on the remote export, not in the local volume.
     /// </summary>
     Task<bool> RemoveStackVolumesAsync(string stackName);
+
+    /// <summary>
+    /// Lists all nodes in the Docker Swarm cluster.
+    /// Must be executed against a Swarm manager node.
+    /// </summary>
+    Task<List<NodeInfo>> ListNodesAsync();
+
+    /// <summary>
+    /// Force-updates a service so all its tasks are restarted and pick up any changed
+    /// secrets or config (equivalent to docker service update --force).
+    /// </summary>
+    Task<bool> ForceUpdateServiceAsync(string serviceName);
+
+    /// <summary>
+    /// Opens a <see cref="MySqlConnection"/> to a remote MySQL server through the
+    /// implementation's transport (e.g. an SSH tunnel).  The caller must dispose
+    /// both the connection <b>and</b> the returned <c>tunnel</c> handle when finished.
+    /// </summary>
+    /// <returns>
+    /// A tuple of (connection, tunnel).  <c>tunnel</c> is <see cref="IDisposable"/>
+    /// and MUST be disposed after the connection is closed.
+    /// </returns>
+    Task<(MySqlConnection Connection, IDisposable Tunnel)> OpenMySqlConnectionAsync(
+        string mysqlHost, int port,
+        string adminUser, string adminPassword);
+
+    /// <summary>
+    /// Executes <c>ALTER USER … IDENTIFIED BY …</c> on a remote MySQL server via
+    /// <see cref="OpenMySqlConnectionAsync"/>.
+    /// </summary>
+    Task<(bool Success, string Error)> AlterMySqlUserPasswordAsync(
+        string mysqlHost, int port,
+        string adminUser, string adminPassword,
+        string targetUser, string newPassword);
+
+    /// <summary>
+    /// Atomically swaps one secret reference on a running service:
+    /// removes <paramref name="oldSecretName"/> and adds <paramref name="newSecretName"/>,
+    /// preserving the in-container path as <paramref name="targetAlias"/> (defaults to
+    /// <paramref name="oldSecretName"/> when null, keeping the same /run/secrets/ filename).
+    /// </summary>
+    Task<bool> ServiceSwapSecretAsync(string serviceName, string oldSecretName, string newSecretName, string? targetAlias = null);
 }
 
 public class StackInfo

OTSSignsOrchestrator.Core/Services/SettingsService.cs

@@ -21,7 +21,7 @@ public class SettingsService
     public const string CatMySql = "MySql";
     public const string CatSmtp = "Smtp";
     public const string CatPangolin = "Pangolin";
-    public const string CatCifs = "Cifs";
+    public const string CatNfs = "Nfs";
     public const string CatDefaults = "Defaults";
 
     // ── Key constants ──────────────────────────────────────────────────────
@@ -49,13 +49,11 @@ public class SettingsService
     // Pangolin
     public const string PangolinEndpoint = "Pangolin.Endpoint";
 
-    // CIFS
-    public const string CifsServer = "Cifs.Server";
-    public const string CifsShareName = "Cifs.ShareName";
-    public const string CifsShareFolder = "Cifs.ShareFolder";
-    public const string CifsUsername = "Cifs.Username";
-    public const string CifsPassword = "Cifs.Password";
-    public const string CifsOptions = "Cifs.Options";
+    // NFS
+    public const string NfsServer = "Nfs.Server";
+    public const string NfsExport = "Nfs.Export";
+    public const string NfsExportFolder = "Nfs.ExportFolder";
+    public const string NfsOptions = "Nfs.Options";
 
     // Instance Defaults
     public const string DefaultCmsImage = "Defaults.CmsImage";