using System.Security.Cryptography;
using System.Text.Json;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Renci.SshNet;
using OTSSignsOrchestrator.Core.Configuration;
using OTSSignsOrchestrator.Core.Services;
using OTSSignsOrchestrator.Server.Clients;
using OTSSignsOrchestrator.Server.Data;
using OTSSignsOrchestrator.Server.Data.Entities;
using OTSSignsOrchestrator.Server.Hubs;
using static OTSSignsOrchestrator.Core.Configuration.AppConstants;
namespace OTSSignsOrchestrator.Server.Workers;
///
/// Phase 1 provisioning pipeline — infrastructure setup. Handles JobType = "provision".
///
/// Steps:
/// 1. mysql-setup — Create DB + user on external MySQL via SSH tunnel
/// 2. docker-secrets — Create Docker Swarm secrets via SSH
/// 3. nfs-dirs — Create NFS sub-directories via SSH
/// 4. authentik-provision — SAML provider, application, groups, invitation flow in Authentik
/// 5. stack-deploy — Render compose YAML and deploy via docker stack deploy
/// 6. credential-store — Store generated credentials in Bitwarden Secrets Manager
///
public sealed class Phase1Pipeline : IProvisioningPipeline
{
public string HandlesJobType => "provision";
private const int TotalSteps = 6;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public Phase1Pipeline(
IServiceProvider services,
ILogger logger)
{
_services = services;
_logger = logger;
}
public async Task ExecuteAsync(Job job, CancellationToken ct)
{
await using var scope = _services.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService();
var hub = scope.ServiceProvider.GetRequiredService>();
var authentikClient = scope.ServiceProvider.GetRequiredService();
var composeRenderer = scope.ServiceProvider.GetRequiredService();
var gitService = scope.ServiceProvider.GetRequiredService();
var bws = scope.ServiceProvider.GetRequiredService();
var settings = scope.ServiceProvider.GetRequiredService();
var ctx = await BuildContextAsync(job, db, ct);
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
var abbrev = ctx.Abbreviation;
var stackName = ctx.DockerStackName;
// Mutable state accumulated across steps
var mysqlPassword = GenerateRandomPassword(32);
var mysqlDbName = $"xibo_{abbrev}";
var mysqlUserName = $"xibo_{abbrev}";
string? authentikProviderId = null;
// ── Step 1: mysql-setup ─────────────────────────────────────────────
await runner.RunAsync("mysql-setup", async () =>
{
var mysqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost");
var mysqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306");
var mysqlAdminUser = await settings.GetAsync(SettingsService.MySqlAdminUser, "root");
var mysqlAdminPassword = await settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
if (!int.TryParse(mysqlPort, out var port)) port = 3306;
// SSH to MySQL host, create database and user
// Reuse pattern from InstanceService.CreateMySqlDatabaseAsync — runs SQL
// via SSH tunnel to the MySQL server.
var sshHost = await GetSwarmSshHostAsync(settings);
using var sshClient = CreateSshClient(sshHost);
sshClient.Connect();
try
{
// Create database
var createDbCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
$"-p'{mysqlAdminPassword}' -e " +
$"\"CREATE DATABASE IF NOT EXISTS \\`{mysqlDbName}\\`\"";
RunSshCommand(sshClient, createDbCmd);
// Create user
var createUserCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
$"-p'{mysqlAdminPassword}' -e " +
$"\"CREATE USER IF NOT EXISTS '{mysqlUserName}'@'%' IDENTIFIED BY '{mysqlPassword}'\"";
RunSshCommand(sshClient, createUserCmd);
// Grant privileges
var grantCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
$"-p'{mysqlAdminPassword}' -e " +
$"\"GRANT ALL PRIVILEGES ON \\`{mysqlDbName}\\`.* TO '{mysqlUserName}'@'%'; FLUSH PRIVILEGES;\"";
RunSshCommand(sshClient, grantCmd);
return $"Database '{mysqlDbName}' and user '{mysqlUserName}' created on {mysqlHost}:{port}.";
}
finally
{
sshClient.Disconnect();
}
}, ct);
// ── Step 2: docker-secrets ──────────────────────────────────────────
await runner.RunAsync("docker-secrets", async () =>
{
// Reuse IDockerSecretsService pattern — create secrets via SSH docker CLI
var sshHost = await GetSwarmSshHostAsync(settings);
using var sshClient = CreateSshClient(sshHost);
sshClient.Connect();
try
{
var secrets = new Dictionary
{
[CustomerMysqlPasswordSecretName(abbrev)] = mysqlPassword,
[CustomerMysqlUserSecretName(abbrev)] = mysqlUserName,
[GlobalMysqlHostSecretName] = await settings.GetAsync(SettingsService.MySqlHost, "localhost"),
[GlobalMysqlPortSecretName] = await settings.GetAsync(SettingsService.MySqlPort, "3306"),
};
var created = new List();
foreach (var (name, value) in secrets)
{
// Remove existing secret if present (idempotent rotate)
RunSshCommandAllowFailure(sshClient, $"docker secret rm {name}");
var safeValue = value.Replace("'", "'\\''");
var cmd = $"printf '%s' '{safeValue}' | docker secret create {name} -";
RunSshCommand(sshClient, cmd);
created.Add(name);
}
return $"Docker secrets created: {string.Join(", ", created)}.";
}
finally
{
sshClient.Disconnect();
}
}, ct);
// ── Step 3: nfs-dirs ────────────────────────────────────────────────
await runner.RunAsync("nfs-dirs", async () =>
{
var nfsServer = await settings.GetAsync(SettingsService.NfsServer);
var nfsExport = await settings.GetAsync(SettingsService.NfsExport);
var nfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder);
if (string.IsNullOrWhiteSpace(nfsServer))
return "NFS server not configured — skipping directory creation.";
var sshHost = await GetSwarmSshHostAsync(settings);
using var sshClient = CreateSshClient(sshHost);
sshClient.Connect();
try
{
// Build the base path for NFS dirs
var export = (nfsExport ?? string.Empty).TrimEnd('/');
var folder = (nfsExportFolder ?? string.Empty).Trim('/');
var basePath = string.IsNullOrEmpty(folder) ? export : $"{export}/{folder}";
var subdirs = new[]
{
$"{abbrev}/cms-custom",
$"{abbrev}/cms-backup",
$"{abbrev}/cms-library",
$"{abbrev}/cms-userscripts",
$"{abbrev}/cms-ca-certs",
};
// Create directories via sudo on the NFS host
// The swarm node mounts NFS, so we can create directories by
// temporarily mounting, creating, then unmounting.
var mountPoint = $"/tmp/nfs-provision-{abbrev}";
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}");
RunSshCommand(sshClient,
$"sudo mount -t nfs4 {nfsServer}:{basePath} {mountPoint}");
try
{
foreach (var subdir in subdirs)
{
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}/{subdir}");
}
}
finally
{
RunSshCommandAllowFailure(sshClient, $"sudo umount {mountPoint}");
RunSshCommandAllowFailure(sshClient, $"sudo rmdir {mountPoint}");
}
return $"NFS directories created under {basePath}: {string.Join(", ", subdirs)}.";
}
finally
{
sshClient.Disconnect();
}
}, ct);
// ── Step 4: authentik-provision ─────────────────────────────────────
await runner.RunAsync("authentik-provision", async () =>
{
var instanceUrl = ctx.InstanceUrl;
var authFlowSlug = await settings.GetAsync(SettingsService.AuthentikAuthorizationFlowSlug, "default-provider-authorization-implicit-consent");
var signingKpId = await settings.GetAsync(SettingsService.AuthentikSigningKeypairId);
// 1. Create SAML provider
var providerResponse = await authentikClient.CreateSamlProviderAsync(new CreateSamlProviderRequest(
Name: $"xibo-{abbrev}",
AuthorizationFlow: authFlowSlug,
AcsUrl: $"{instanceUrl}/saml/acs",
Issuer: $"xibo-{abbrev}",
SpBinding: "post",
Audience: instanceUrl,
SigningKp: signingKpId));
EnsureSuccess(providerResponse);
var providerData = providerResponse.Content
?? throw new InvalidOperationException("Authentik SAML provider creation returned empty response.");
authentikProviderId = providerData["pk"]?.ToString();
// 2. Create Authentik application linked to provider
var appResponse = await authentikClient.CreateApplicationAsync(new CreateAuthentikApplicationRequest(
Name: $"xibo-{abbrev}",
Slug: $"xibo-{abbrev}",
Provider: authentikProviderId!,
MetaLaunchUrl: instanceUrl));
EnsureSuccess(appResponse);
// 3. Create 4 tenant groups
var groupNames = new[]
{
$"customer-{abbrev}",
$"customer-{abbrev}-viewer",
$"customer-{abbrev}-editor",
$"customer-{abbrev}-admin",
};
foreach (var groupName in groupNames)
{
var groupResp = await authentikClient.CreateGroupAsync(new CreateAuthentikGroupRequest(
Name: groupName,
IsSuperuser: false,
Parent: null));
EnsureSuccess(groupResp);
}
// 4. Create invitation flow
var inviteResponse = await authentikClient.CreateInvitationAsync(new CreateFlowRequest(
Name: $"invite-{abbrev}",
SingleUse: true,
Expires: DateTimeOffset.UtcNow.AddDays(30)));
EnsureSuccess(inviteResponse);
// Store Authentik provider ID on the Instance entity
var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct);
if (instance is not null)
{
instance.AuthentikProviderId = authentikProviderId;
await db.SaveChangesAsync(ct);
}
return $"Authentik SAML provider '{authentikProviderId}', application 'xibo-{abbrev}', " +
$"4 groups, and invitation flow created.";
}, ct);
// ── Step 5: stack-deploy ────────────────────────────────────────────
await runner.RunAsync("stack-deploy", async () =>
{
// Fetch template
var repoUrl = await settings.GetAsync(SettingsService.GitRepoUrl);
var repoPat = await settings.GetAsync(SettingsService.GitRepoPat);
if (string.IsNullOrWhiteSpace(repoUrl))
throw new InvalidOperationException("Git template repository URL is not configured.");
var templateConfig = await gitService.FetchAsync(repoUrl, repoPat);
// Build render context
var renderCtx = new RenderContext
{
CustomerName = ctx.CompanyName,
CustomerAbbrev = abbrev,
StackName = stackName,
CmsServerName = await settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "app.ots-signs.com"),
HostHttpPort = 80,
CmsImage = await settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3"),
MemcachedImage = await settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine"),
QuickChartImage = await settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart"),
NewtImage = await settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt"),
ThemeHostPath = await settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme"),
MySqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost"),
MySqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306"),
MySqlDatabase = mysqlDbName,
MySqlUser = mysqlUserName,
MySqlPassword = mysqlPassword,
SmtpServer = await settings.GetAsync(SettingsService.SmtpServer, string.Empty),
SmtpUsername = await settings.GetAsync(SettingsService.SmtpUsername, string.Empty),
SmtpPassword = await settings.GetAsync(SettingsService.SmtpPassword, string.Empty),
SmtpUseTls = await settings.GetAsync(SettingsService.SmtpUseTls, "YES"),
SmtpUseStartTls = await settings.GetAsync(SettingsService.SmtpUseStartTls, "YES"),
SmtpRewriteDomain = await settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty),
SmtpHostname = await settings.GetAsync(SettingsService.SmtpHostname, string.Empty),
SmtpFromLineOverride = await settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO"),
PhpPostMaxSize = await settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G"),
PhpUploadMaxFilesize = await settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G"),
PhpMaxExecutionTime = await settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600"),
PangolinEndpoint = await settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net"),
NfsServer = await settings.GetAsync(SettingsService.NfsServer),
NfsExport = await settings.GetAsync(SettingsService.NfsExport),
NfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder),
NfsExtraOptions = await settings.GetAsync(SettingsService.NfsOptions, string.Empty),
};
var composeYaml = composeRenderer.Render(templateConfig.Yaml, renderCtx);
// Deploy via SSH: pipe compose YAML to docker stack deploy
var sshHost = await GetSwarmSshHostAsync(settings);
using var sshClient = CreateSshClient(sshHost);
sshClient.Connect();
try
{
var safeYaml = composeYaml.Replace("'", "'\\''");
var deployCmd = $"printf '%s' '{safeYaml}' | docker stack deploy -c - {stackName}";
var result = RunSshCommand(sshClient, deployCmd);
return $"Stack '{stackName}' deployed. Output: {result}";
}
finally
{
sshClient.Disconnect();
}
}, ct);
// ── Step 6: credential-store ────────────────────────────────────────
await runner.RunAsync("credential-store", async () =>
{
// Bitwarden item structure:
// Key: "{abbrev}/mysql-password" → MySQL password for xibo_{abbrev} user
// Key: "{abbrev}/mysql-database" → Database name (for reference)
// Key: "{abbrev}/mysql-user" → MySQL username (for reference)
// All stored in the instance Bitwarden project via IBitwardenSecretService.
await bws.CreateInstanceSecretAsync(
key: $"{abbrev}/mysql-password",
value: mysqlPassword,
note: $"MySQL password for instance {abbrev}. DB: {mysqlDbName}, User: {mysqlUserName}");
// Also persist in settings for Phase2 and future redeploys
await settings.SetAsync(
SettingsService.InstanceMySqlPassword(abbrev),
mysqlPassword,
SettingsService.CatInstance,
isSensitive: true);
// Clear in-memory password
mysqlPassword = string.Empty;
return $"Credentials stored in Bitwarden for instance '{abbrev}'.";
}, ct);
_logger.LogInformation("Phase1Pipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
}
// ─────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────
private static async Task BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
{
var customer = await db.Customers
.Include(c => c.Instances)
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
var instance = customer.Instances.FirstOrDefault()
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
var abbrev = customer.Abbreviation.ToLowerInvariant();
return new PipelineContext
{
JobId = job.Id,
CustomerId = customer.Id,
InstanceId = instance.Id,
Abbreviation = abbrev,
CompanyName = customer.CompanyName,
AdminEmail = customer.AdminEmail,
AdminFirstName = customer.AdminFirstName,
InstanceUrl = instance.XiboUrl,
DockerStackName = instance.DockerStackName,
ParametersJson = job.Parameters,
};
}
///
/// Reads SSH connection details for the Docker Swarm host from settings.
/// Expects settings keys: "Ssh.SwarmHost", "Ssh.SwarmPort", "Ssh.SwarmUser", "Ssh.SwarmKeyPath".
///
private static async Task GetSwarmSshHostAsync(SettingsService settings)
{
var host = await settings.GetAsync("Ssh.SwarmHost")
?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost).");
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
var password = await settings.GetAsync("Ssh.SwarmPassword");
if (!int.TryParse(portStr, out var port)) port = 22;
return new SshConnectionInfo(host, port, user, keyPath, password);
}
private static SshClient CreateSshClient(SshConnectionInfo info)
{
var authMethods = new List();
if (!string.IsNullOrEmpty(info.KeyPath))
{
authMethods.Add(new PrivateKeyAuthenticationMethod(
info.Username, new PrivateKeyFile(info.KeyPath)));
}
if (!string.IsNullOrEmpty(info.Password))
{
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
}
if (authMethods.Count == 0)
{
var defaultKeyPath = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
if (File.Exists(defaultKeyPath))
{
authMethods.Add(new PrivateKeyAuthenticationMethod(
info.Username, new PrivateKeyFile(defaultKeyPath)));
}
else
{
throw new InvalidOperationException(
$"No SSH authentication method available for {info.Host}:{info.Port}. " +
"Configure Ssh.SwarmKeyPath or Ssh.SwarmPassword in settings.");
}
}
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
return new SshClient(connInfo);
}
private static string RunSshCommand(SshClient client, string command)
{
using var cmd = client.RunCommand(command);
if (cmd.ExitStatus != 0)
throw new InvalidOperationException(
$"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
return cmd.Result;
}
private static void RunSshCommandAllowFailure(SshClient client, string command)
{
using var cmd = client.RunCommand(command);
// Intentionally ignore exit code — used for idempotent cleanup operations
}
private static string GenerateRandomPassword(int length)
{
const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*";
return RandomNumberGenerator.GetString(chars, length);
}
private static void EnsureSuccess(Refit.IApiResponse response)
{
if (!response.IsSuccessStatusCode)
throw new InvalidOperationException(
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
}
/// SSH connection details read from Bitwarden settings.
internal sealed record SshConnectionInfo(
string Host,
int Port,
string Username,
string? KeyPath,
string? Password);
}