using System.Security.Cryptography; using System.Text.Json; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Renci.SshNet; using OTSSignsOrchestrator.Core.Configuration; using OTSSignsOrchestrator.Core.Services; using OTSSignsOrchestrator.Server.Clients; using OTSSignsOrchestrator.Server.Data; using OTSSignsOrchestrator.Server.Data.Entities; using OTSSignsOrchestrator.Server.Hubs; using static OTSSignsOrchestrator.Core.Configuration.AppConstants; namespace OTSSignsOrchestrator.Server.Workers; /// /// Phase 1 provisioning pipeline — infrastructure setup. Handles JobType = "provision". /// /// Steps: /// 1. mysql-setup — Create DB + user on external MySQL via SSH tunnel /// 2. docker-secrets — Create Docker Swarm secrets via SSH /// 3. nfs-dirs — Create NFS sub-directories via SSH /// 4. authentik-provision — SAML provider, application, groups, invitation flow in Authentik /// 5. stack-deploy — Render compose YAML and deploy via docker stack deploy /// 6. credential-store — Store generated credentials in Bitwarden Secrets Manager /// public sealed class Phase1Pipeline : IProvisioningPipeline { public string HandlesJobType => "provision"; private const int TotalSteps = 6; private readonly IServiceProvider _services; private readonly ILogger _logger; public Phase1Pipeline( IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } public async Task ExecuteAsync(Job job, CancellationToken ct) { await using var scope = _services.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); var hub = scope.ServiceProvider.GetRequiredService>(); var authentikClient = scope.ServiceProvider.GetRequiredService(); var composeRenderer = scope.ServiceProvider.GetRequiredService(); var gitService = scope.ServiceProvider.GetRequiredService(); var bws = scope.ServiceProvider.GetRequiredService(); var settings = scope.ServiceProvider.GetRequiredService(); var ctx = await BuildContextAsync(job, db, ct); var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps); var abbrev = ctx.Abbreviation; var stackName = ctx.DockerStackName; // Mutable state accumulated across steps var mysqlPassword = GenerateRandomPassword(32); var mysqlDbName = $"xibo_{abbrev}"; var mysqlUserName = $"xibo_{abbrev}"; string? authentikProviderId = null; // ── Step 1: mysql-setup ───────────────────────────────────────────── await runner.RunAsync("mysql-setup", async () => { var mysqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost"); var mysqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306"); var mysqlAdminUser = await settings.GetAsync(SettingsService.MySqlAdminUser, "root"); var mysqlAdminPassword = await settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty); if (!int.TryParse(mysqlPort, out var port)) port = 3306; // SSH to MySQL host, create database and user // Reuse pattern from InstanceService.CreateMySqlDatabaseAsync — runs SQL // via SSH tunnel to the MySQL server. var sshHost = await GetSwarmSshHostAsync(settings); using var sshClient = CreateSshClient(sshHost); sshClient.Connect(); try { // Create database var createDbCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " + $"-p'{mysqlAdminPassword}' -e " + $"\"CREATE DATABASE IF NOT EXISTS \\`{mysqlDbName}\\`\""; RunSshCommand(sshClient, createDbCmd); // Create user var createUserCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " + $"-p'{mysqlAdminPassword}' -e " + $"\"CREATE USER IF NOT EXISTS '{mysqlUserName}'@'%' IDENTIFIED BY '{mysqlPassword}'\""; RunSshCommand(sshClient, createUserCmd); // Grant privileges var grantCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " + $"-p'{mysqlAdminPassword}' -e " + $"\"GRANT ALL PRIVILEGES ON \\`{mysqlDbName}\\`.* TO '{mysqlUserName}'@'%'; FLUSH PRIVILEGES;\""; RunSshCommand(sshClient, grantCmd); return $"Database '{mysqlDbName}' and user '{mysqlUserName}' created on {mysqlHost}:{port}."; } finally { sshClient.Disconnect(); } }, ct); // ── Step 2: docker-secrets ────────────────────────────────────────── await runner.RunAsync("docker-secrets", async () => { // Reuse IDockerSecretsService pattern — create secrets via SSH docker CLI var sshHost = await GetSwarmSshHostAsync(settings); using var sshClient = CreateSshClient(sshHost); sshClient.Connect(); try { var secrets = new Dictionary { [CustomerMysqlPasswordSecretName(abbrev)] = mysqlPassword, [CustomerMysqlUserSecretName(abbrev)] = mysqlUserName, [GlobalMysqlHostSecretName] = await settings.GetAsync(SettingsService.MySqlHost, "localhost"), [GlobalMysqlPortSecretName] = await settings.GetAsync(SettingsService.MySqlPort, "3306"), }; var created = new List(); foreach (var (name, value) in secrets) { // Remove existing secret if present (idempotent rotate) RunSshCommandAllowFailure(sshClient, $"docker secret rm {name}"); var safeValue = value.Replace("'", "'\\''"); var cmd = $"printf '%s' '{safeValue}' | docker secret create {name} -"; RunSshCommand(sshClient, cmd); created.Add(name); } return $"Docker secrets created: {string.Join(", ", created)}."; } finally { sshClient.Disconnect(); } }, ct); // ── Step 3: nfs-dirs ──────────────────────────────────────────────── await runner.RunAsync("nfs-dirs", async () => { var nfsServer = await settings.GetAsync(SettingsService.NfsServer); var nfsExport = await settings.GetAsync(SettingsService.NfsExport); var nfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder); if (string.IsNullOrWhiteSpace(nfsServer)) return "NFS server not configured — skipping directory creation."; var sshHost = await GetSwarmSshHostAsync(settings); using var sshClient = CreateSshClient(sshHost); sshClient.Connect(); try { // Build the base path for NFS dirs var export = (nfsExport ?? string.Empty).TrimEnd('/'); var folder = (nfsExportFolder ?? string.Empty).Trim('/'); var basePath = string.IsNullOrEmpty(folder) ? export : $"{export}/{folder}"; var subdirs = new[] { $"{abbrev}/cms-custom", $"{abbrev}/cms-backup", $"{abbrev}/cms-library", $"{abbrev}/cms-userscripts", $"{abbrev}/cms-ca-certs", }; // Create directories via sudo on the NFS host // The swarm node mounts NFS, so we can create directories by // temporarily mounting, creating, then unmounting. var mountPoint = $"/tmp/nfs-provision-{abbrev}"; RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}"); RunSshCommand(sshClient, $"sudo mount -t nfs4 {nfsServer}:{basePath} {mountPoint}"); try { foreach (var subdir in subdirs) { RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}/{subdir}"); } } finally { RunSshCommandAllowFailure(sshClient, $"sudo umount {mountPoint}"); RunSshCommandAllowFailure(sshClient, $"sudo rmdir {mountPoint}"); } return $"NFS directories created under {basePath}: {string.Join(", ", subdirs)}."; } finally { sshClient.Disconnect(); } }, ct); // ── Step 4: authentik-provision ───────────────────────────────────── await runner.RunAsync("authentik-provision", async () => { var instanceUrl = ctx.InstanceUrl; var authFlowSlug = await settings.GetAsync(SettingsService.AuthentikAuthorizationFlowSlug, "default-provider-authorization-implicit-consent"); var signingKpId = await settings.GetAsync(SettingsService.AuthentikSigningKeypairId); // 1. Create SAML provider var providerResponse = await authentikClient.CreateSamlProviderAsync(new CreateSamlProviderRequest( Name: $"xibo-{abbrev}", AuthorizationFlow: authFlowSlug, AcsUrl: $"{instanceUrl}/saml/acs", Issuer: $"xibo-{abbrev}", SpBinding: "post", Audience: instanceUrl, SigningKp: signingKpId)); EnsureSuccess(providerResponse); var providerData = providerResponse.Content ?? throw new InvalidOperationException("Authentik SAML provider creation returned empty response."); authentikProviderId = providerData["pk"]?.ToString(); // 2. Create Authentik application linked to provider var appResponse = await authentikClient.CreateApplicationAsync(new CreateAuthentikApplicationRequest( Name: $"xibo-{abbrev}", Slug: $"xibo-{abbrev}", Provider: authentikProviderId!, MetaLaunchUrl: instanceUrl)); EnsureSuccess(appResponse); // 3. Create 4 tenant groups var groupNames = new[] { $"customer-{abbrev}", $"customer-{abbrev}-viewer", $"customer-{abbrev}-editor", $"customer-{abbrev}-admin", }; foreach (var groupName in groupNames) { var groupResp = await authentikClient.CreateGroupAsync(new CreateAuthentikGroupRequest( Name: groupName, IsSuperuser: false, Parent: null)); EnsureSuccess(groupResp); } // 4. Create invitation flow var inviteResponse = await authentikClient.CreateInvitationAsync(new CreateFlowRequest( Name: $"invite-{abbrev}", SingleUse: true, Expires: DateTimeOffset.UtcNow.AddDays(30))); EnsureSuccess(inviteResponse); // Store Authentik provider ID on the Instance entity var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct); if (instance is not null) { instance.AuthentikProviderId = authentikProviderId; await db.SaveChangesAsync(ct); } return $"Authentik SAML provider '{authentikProviderId}', application 'xibo-{abbrev}', " + $"4 groups, and invitation flow created."; }, ct); // ── Step 5: stack-deploy ──────────────────────────────────────────── await runner.RunAsync("stack-deploy", async () => { // Fetch template var repoUrl = await settings.GetAsync(SettingsService.GitRepoUrl); var repoPat = await settings.GetAsync(SettingsService.GitRepoPat); if (string.IsNullOrWhiteSpace(repoUrl)) throw new InvalidOperationException("Git template repository URL is not configured."); var templateConfig = await gitService.FetchAsync(repoUrl, repoPat); // Build render context var renderCtx = new RenderContext { CustomerName = ctx.CompanyName, CustomerAbbrev = abbrev, StackName = stackName, CmsServerName = await settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "app.ots-signs.com"), HostHttpPort = 80, CmsImage = await settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3"), MemcachedImage = await settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine"), QuickChartImage = await settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart"), NewtImage = await settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt"), ThemeHostPath = await settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme"), MySqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost"), MySqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306"), MySqlDatabase = mysqlDbName, MySqlUser = mysqlUserName, MySqlPassword = mysqlPassword, SmtpServer = await settings.GetAsync(SettingsService.SmtpServer, string.Empty), SmtpUsername = await settings.GetAsync(SettingsService.SmtpUsername, string.Empty), SmtpPassword = await settings.GetAsync(SettingsService.SmtpPassword, string.Empty), SmtpUseTls = await settings.GetAsync(SettingsService.SmtpUseTls, "YES"), SmtpUseStartTls = await settings.GetAsync(SettingsService.SmtpUseStartTls, "YES"), SmtpRewriteDomain = await settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty), SmtpHostname = await settings.GetAsync(SettingsService.SmtpHostname, string.Empty), SmtpFromLineOverride = await settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO"), PhpPostMaxSize = await settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G"), PhpUploadMaxFilesize = await settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G"), PhpMaxExecutionTime = await settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600"), PangolinEndpoint = await settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net"), NfsServer = await settings.GetAsync(SettingsService.NfsServer), NfsExport = await settings.GetAsync(SettingsService.NfsExport), NfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder), NfsExtraOptions = await settings.GetAsync(SettingsService.NfsOptions, string.Empty), }; var composeYaml = composeRenderer.Render(templateConfig.Yaml, renderCtx); // Deploy via SSH: pipe compose YAML to docker stack deploy var sshHost = await GetSwarmSshHostAsync(settings); using var sshClient = CreateSshClient(sshHost); sshClient.Connect(); try { var safeYaml = composeYaml.Replace("'", "'\\''"); var deployCmd = $"printf '%s' '{safeYaml}' | docker stack deploy -c - {stackName}"; var result = RunSshCommand(sshClient, deployCmd); return $"Stack '{stackName}' deployed. Output: {result}"; } finally { sshClient.Disconnect(); } }, ct); // ── Step 6: credential-store ──────────────────────────────────────── await runner.RunAsync("credential-store", async () => { // Bitwarden item structure: // Key: "{abbrev}/mysql-password" → MySQL password for xibo_{abbrev} user // Key: "{abbrev}/mysql-database" → Database name (for reference) // Key: "{abbrev}/mysql-user" → MySQL username (for reference) // All stored in the instance Bitwarden project via IBitwardenSecretService. await bws.CreateInstanceSecretAsync( key: $"{abbrev}/mysql-password", value: mysqlPassword, note: $"MySQL password for instance {abbrev}. DB: {mysqlDbName}, User: {mysqlUserName}"); // Also persist in settings for Phase2 and future redeploys await settings.SetAsync( SettingsService.InstanceMySqlPassword(abbrev), mysqlPassword, SettingsService.CatInstance, isSensitive: true); // Clear in-memory password mysqlPassword = string.Empty; return $"Credentials stored in Bitwarden for instance '{abbrev}'."; }, ct); _logger.LogInformation("Phase1Pipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev); } // ───────────────────────────────────────────────────────────────────────── // Helpers // ───────────────────────────────────────────────────────────────────────── private static async Task BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct) { var customer = await db.Customers .Include(c => c.Instances) .FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct) ?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}."); var instance = customer.Instances.FirstOrDefault() ?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}."); var abbrev = customer.Abbreviation.ToLowerInvariant(); return new PipelineContext { JobId = job.Id, CustomerId = customer.Id, InstanceId = instance.Id, Abbreviation = abbrev, CompanyName = customer.CompanyName, AdminEmail = customer.AdminEmail, AdminFirstName = customer.AdminFirstName, InstanceUrl = instance.XiboUrl, DockerStackName = instance.DockerStackName, ParametersJson = job.Parameters, }; } /// /// Reads SSH connection details for the Docker Swarm host from settings. /// Expects settings keys: "Ssh.SwarmHost", "Ssh.SwarmPort", "Ssh.SwarmUser", "Ssh.SwarmKeyPath". /// private static async Task GetSwarmSshHostAsync(SettingsService settings) { var host = await settings.GetAsync("Ssh.SwarmHost") ?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost)."); var portStr = await settings.GetAsync("Ssh.SwarmPort", "22"); var user = await settings.GetAsync("Ssh.SwarmUser", "root"); var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath"); var password = await settings.GetAsync("Ssh.SwarmPassword"); if (!int.TryParse(portStr, out var port)) port = 22; return new SshConnectionInfo(host, port, user, keyPath, password); } private static SshClient CreateSshClient(SshConnectionInfo info) { var authMethods = new List(); if (!string.IsNullOrEmpty(info.KeyPath)) { authMethods.Add(new PrivateKeyAuthenticationMethod( info.Username, new PrivateKeyFile(info.KeyPath))); } if (!string.IsNullOrEmpty(info.Password)) { authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password)); } if (authMethods.Count == 0) { var defaultKeyPath = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa"); if (File.Exists(defaultKeyPath)) { authMethods.Add(new PrivateKeyAuthenticationMethod( info.Username, new PrivateKeyFile(defaultKeyPath))); } else { throw new InvalidOperationException( $"No SSH authentication method available for {info.Host}:{info.Port}. " + "Configure Ssh.SwarmKeyPath or Ssh.SwarmPassword in settings."); } } var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray()); return new SshClient(connInfo); } private static string RunSshCommand(SshClient client, string command) { using var cmd = client.RunCommand(command); if (cmd.ExitStatus != 0) throw new InvalidOperationException( $"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}"); return cmd.Result; } private static void RunSshCommandAllowFailure(SshClient client, string command) { using var cmd = client.RunCommand(command); // Intentionally ignore exit code — used for idempotent cleanup operations } private static string GenerateRandomPassword(int length) { const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*"; return RandomNumberGenerator.GetString(chars, length); } private static void EnsureSuccess(Refit.IApiResponse response) { if (!response.IsSuccessStatusCode) throw new InvalidOperationException( $"API call failed with status {response.StatusCode}: {response.Error?.Content}"); } /// SSH connection details read from Bitwarden settings. internal sealed record SshConnectionInfo( string Host, int Port, string Username, string? KeyPath, string? Password); }