feat: Implement provisioning pipelines for subscription management
- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes. - Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging. - Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR. - Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes. - Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots. - Introduce XiboFeatureManifests for hardcoded feature ACLs per role. - Add docker-compose.dev.yml for local development with PostgreSQL setup.
This commit is contained in:
Matt Batchelder
207
OTSSignsOrchestrator.Server/Workers/ByoiSamlPipeline.cs
Normal file
207
OTSSignsOrchestrator.Server/Workers/ByoiSamlPipeline.cs
Normal file
@@ -0,0 +1,207 @@
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Quartz;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
using OTSSignsOrchestrator.Server.Jobs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// BYOI SAML provisioning pipeline — configures an upstream SAML source in Authentik
|
||||
/// so a Pro-tier customer can federate their own Identity Provider.
|
||||
///
|
||||
/// Handles <c>JobType = "provision-byoi"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. import-cert — Import the customer's public cert PEM into Authentik
|
||||
/// 2. create-saml-source — Create the upstream SAML source in Authentik
|
||||
/// 3. store-metadata — Persist slug, entity_id, sso_url, cert info to ByoiConfig
|
||||
/// 4. schedule-cert-monitor — Register a Quartz ByoiCertExpiryJob for daily cert expiry checks
|
||||
///
|
||||
/// IMPORTANT: Xibo's settings-custom.php is IDENTICAL for Pro BYOI and Essentials.
|
||||
/// Xibo always authenticates through Authentik. BYOI is entirely handled within
|
||||
/// Authentik's upstream SAML Source config — no Xibo changes are needed.
|
||||
/// </summary>
|
||||
public sealed class ByoiSamlPipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "provision-byoi";
|
||||
|
||||
private const int TotalSteps = 4;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<ByoiSamlPipeline> _logger;
|
||||
|
||||
public ByoiSamlPipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<ByoiSamlPipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var authentikClient = scope.ServiceProvider.GetRequiredService<IAuthentikClient>();
|
||||
var authentikOpts = scope.ServiceProvider.GetRequiredService<IOptions<AuthentikOptions>>().Value;
|
||||
var schedulerFactory = scope.ServiceProvider.GetRequiredService<ISchedulerFactory>();
|
||||
|
||||
// Load customer + instance
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found.");
|
||||
|
||||
if (customer.Plan != CustomerPlan.Pro)
|
||||
throw new InvalidOperationException("BYOI SAML is only available for Pro tier customers.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation;
|
||||
var companyName = customer.CompanyName;
|
||||
|
||||
// Parse BYOI parameters from Job.Parameters JSON
|
||||
var parms = JsonSerializer.Deserialize<ByoiParameters>(
|
||||
job.Parameters ?? throw new InvalidOperationException("Job.Parameters is null."),
|
||||
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })
|
||||
?? throw new InvalidOperationException("Failed to deserialize BYOI parameters.");
|
||||
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
// Mutable state accumulated across steps
|
||||
string? verificationKpId = null;
|
||||
string slug = $"byoi-{abbrev}";
|
||||
DateTime certExpiry;
|
||||
|
||||
// Parse the cert up front to get expiry and validate
|
||||
using var cert = X509CertificateLoader.LoadCertificate(
|
||||
Convert.FromBase64String(ExtractBase64FromPem(parms.CustomerCertPem)));
|
||||
certExpiry = cert.NotAfter.ToUniversalTime();
|
||||
|
||||
// ── Step 1: import-cert ─────────────────────────────────────────────
|
||||
await runner.RunAsync("import-cert", async () =>
|
||||
{
|
||||
var response = await authentikClient.ImportCertificateAsync(new ImportCertRequest(
|
||||
Name: $"byoi-{abbrev}-cert",
|
||||
CertificateData: parms.CustomerCertPem,
|
||||
KeyData: null)); // PUBLIC cert only — no private key
|
||||
|
||||
if (!response.IsSuccessStatusCode || response.Content is null)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to import certificate: {response.Error?.Content ?? response.ReasonPhrase}");
|
||||
|
||||
verificationKpId = response.Content["pk"]?.ToString()
|
||||
?? throw new InvalidOperationException("Authentik cert import returned no pk.");
|
||||
|
||||
return $"Imported customer cert as keypair '{verificationKpId}'.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: create-saml-source ──────────────────────────────────────
|
||||
await runner.RunAsync("create-saml-source", async () =>
|
||||
{
|
||||
var response = await authentikClient.CreateSamlSourceAsync(new CreateSamlSourceRequest(
|
||||
Name: $"{companyName} IdP",
|
||||
Slug: slug,
|
||||
SsoUrl: parms.CustomerSsoUrl,
|
||||
SloUrl: parms.CustomerSloUrl,
|
||||
Issuer: parms.CustomerIdpEntityId,
|
||||
SigningKp: string.IsNullOrWhiteSpace(authentikOpts.OtsSigningKpId) ? null : authentikOpts.OtsSigningKpId,
|
||||
VerificationKp: verificationKpId,
|
||||
BindingType: "redirect",
|
||||
NameIdPolicy: "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress",
|
||||
PreAuthenticationFlow: authentikOpts.SourcePreAuthFlowSlug,
|
||||
AuthenticationFlow: authentikOpts.SourceAuthFlowSlug,
|
||||
AllowIdpInitiated: false)); // REQUIRED: IdP-initiated SSO is a CSRF risk
|
||||
|
||||
if (!response.IsSuccessStatusCode || response.Content is null)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to create SAML source: {response.Error?.Content ?? response.ReasonPhrase}");
|
||||
|
||||
return $"SAML source '{slug}' created with AllowIdpInitiated=false.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: store-metadata ──────────────────────────────────────────
|
||||
await runner.RunAsync("store-metadata", async () =>
|
||||
{
|
||||
var byoiConfig = new ByoiConfig
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = instance.Id,
|
||||
Slug = slug,
|
||||
EntityId = parms.CustomerIdpEntityId,
|
||||
SsoUrl = parms.CustomerSsoUrl,
|
||||
CertPem = parms.CustomerCertPem,
|
||||
CertExpiry = certExpiry,
|
||||
Enabled = true,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
db.ByoiConfigs.Add(byoiConfig);
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"ByoiConfig stored: slug={slug}, certExpiry={certExpiry:O}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: schedule-cert-monitor ───────────────────────────────────
|
||||
await runner.RunAsync("schedule-cert-monitor", async () =>
|
||||
{
|
||||
var scheduler = await schedulerFactory.GetScheduler(ct);
|
||||
var jobKey = new JobKey($"byoi-cert-expiry-{abbrev}", "byoi-cert-expiry");
|
||||
|
||||
// Only schedule if not already present (idempotent)
|
||||
if (!await scheduler.CheckExists(jobKey, ct))
|
||||
{
|
||||
var quartzJob = JobBuilder.Create<ByoiCertExpiryJob>()
|
||||
.WithIdentity(jobKey)
|
||||
.UsingJobData("instanceId", instance.Id.ToString())
|
||||
.Build();
|
||||
|
||||
var trigger = TriggerBuilder.Create()
|
||||
.WithIdentity($"byoi-cert-expiry-{abbrev}-trigger", "byoi-cert-expiry")
|
||||
.WithDailyTimeIntervalSchedule(s => s
|
||||
.OnEveryDay()
|
||||
.StartingDailyAt(TimeOfDay.HourAndMinuteOfDay(6, 0))
|
||||
.WithRepeatCount(1))
|
||||
.StartNow()
|
||||
.Build();
|
||||
|
||||
await scheduler.ScheduleJob(quartzJob, trigger, ct);
|
||||
}
|
||||
|
||||
return $"Quartz ByoiCertExpiryJob scheduled daily for instance {instance.Id}.";
|
||||
}, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the Base64 payload from a PEM string, stripping headers/footers.
|
||||
/// </summary>
|
||||
private static string ExtractBase64FromPem(string pem)
|
||||
{
|
||||
return pem
|
||||
.Replace("-----BEGIN CERTIFICATE-----", "")
|
||||
.Replace("-----END CERTIFICATE-----", "")
|
||||
.Replace("\r", "")
|
||||
.Replace("\n", "")
|
||||
.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deserialized from Job.Parameters JSON for provision-byoi jobs.
|
||||
/// </summary>
|
||||
public sealed record ByoiParameters
|
||||
{
|
||||
public string CustomerCertPem { get; init; } = string.Empty;
|
||||
public string CustomerSsoUrl { get; init; } = string.Empty;
|
||||
public string CustomerIdpEntityId { get; init; } = string.Empty;
|
||||
public string? CustomerSloUrl { get; init; }
|
||||
}
|
||||
430
OTSSignsOrchestrator.Server/Workers/DecommissionPipeline.cs
Normal file
430
OTSSignsOrchestrator.Server/Workers/DecommissionPipeline.cs
Normal file
@@ -0,0 +1,430 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Renci.SshNet;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Full decommission pipeline — removes all infrastructure for a cancelled subscription.
|
||||
/// Handles <c>JobType = "decommission"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. stack-remove — docker stack rm xibo-{abbrev}
|
||||
/// 2. authentik-cleanup — Delete SAML provider, application, 4 tenant groups (+BYOI source)
|
||||
/// 3. oauth2-cleanup — Delete Xibo OAuth2 application via API
|
||||
/// 4. mysql-cleanup — DROP DATABASE + DROP USER via SSH
|
||||
/// 5. nfs-archive — mv /nfs/{abbrev} → /nfs/archived/{abbrev}-{timestamp} (retain 30d min)
|
||||
/// 6. registry-update — Customer.Status = Decommissioned, Instance health = Critical,
|
||||
/// final AuditLog, broadcast InstanceStatusChanged
|
||||
/// </summary>
|
||||
public sealed class DecommissionPipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "decommission";
|
||||
|
||||
private const int TotalSteps = 6;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<DecommissionPipeline> _logger;
|
||||
|
||||
public DecommissionPipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<DecommissionPipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var authentikClient = scope.ServiceProvider.GetRequiredService<IAuthentikClient>();
|
||||
var xiboFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
var stackName = ctx.DockerStackName;
|
||||
|
||||
// ── Step 1: stack-remove ────────────────────────────────────────────
|
||||
await runner.RunAsync("stack-remove", async () =>
|
||||
{
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
var result = RunSshCommand(sshClient, $"docker stack rm {stackName}");
|
||||
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/decommission",
|
||||
Action = "stack-remove",
|
||||
Target = stackName,
|
||||
Outcome = "success",
|
||||
Detail = $"Docker stack '{stackName}' removed. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"Docker stack '{stackName}' removed. Output: {result}";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: authentik-cleanup ───────────────────────────────────────
|
||||
await runner.RunAsync("authentik-cleanup", async () =>
|
||||
{
|
||||
var instance = await db.Instances
|
||||
.Include(i => i.ByoiConfigs)
|
||||
.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct)
|
||||
?? throw new InvalidOperationException($"Instance {ctx.InstanceId} not found.");
|
||||
|
||||
var cleaned = new List<string>();
|
||||
|
||||
// Delete SAML provider (stored on Instance.AuthentikProviderId)
|
||||
if (!string.IsNullOrEmpty(instance.AuthentikProviderId)
|
||||
&& int.TryParse(instance.AuthentikProviderId, out var providerId))
|
||||
{
|
||||
await authentikClient.DeleteSamlProviderAsync(providerId);
|
||||
cleaned.Add($"SAML provider {providerId}");
|
||||
}
|
||||
|
||||
// Delete Authentik application
|
||||
await authentikClient.DeleteApplicationAsync($"xibo-{abbrev}");
|
||||
cleaned.Add($"application xibo-{abbrev}");
|
||||
|
||||
// Delete 4 tenant groups — search by name prefix, match by exact name
|
||||
var groupNames = new[]
|
||||
{
|
||||
$"customer-{abbrev}",
|
||||
$"customer-{abbrev}-viewer",
|
||||
$"customer-{abbrev}-editor",
|
||||
$"customer-{abbrev}-admin",
|
||||
};
|
||||
|
||||
var searchResp = await authentikClient.ListGroupsAsync(search: $"customer-{abbrev}");
|
||||
if (searchResp.IsSuccessStatusCode && searchResp.Content?.Results is { } groups)
|
||||
{
|
||||
foreach (var groupName in groupNames)
|
||||
{
|
||||
var match = groups.FirstOrDefault(g =>
|
||||
g.TryGetValue("name", out var n) && n?.ToString() == groupName);
|
||||
|
||||
if (match is not null && match.TryGetValue("pk", out var pk))
|
||||
{
|
||||
await authentikClient.DeleteGroupAsync(pk.ToString()!);
|
||||
cleaned.Add($"group {groupName}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If BYOI was enabled, delete the SAML source
|
||||
var byoiConfig = instance.ByoiConfigs.FirstOrDefault(b => b.Enabled);
|
||||
if (byoiConfig is not null)
|
||||
{
|
||||
await authentikClient.DeleteSamlSourceAsync($"byoi-{abbrev}");
|
||||
cleaned.Add($"BYOI SAML source byoi-{abbrev}");
|
||||
}
|
||||
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/decommission",
|
||||
Action = "authentik-cleanup",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"Cleaned up: {string.Join(", ", cleaned)}. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"Authentik cleanup completed: {string.Join(", ", cleaned)}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: oauth2-cleanup ──────────────────────────────────────────
|
||||
await runner.RunAsync("oauth2-cleanup", async () =>
|
||||
{
|
||||
var oauthReg = await db.OauthAppRegistries
|
||||
.Where(r => r.InstanceId == ctx.InstanceId)
|
||||
.OrderByDescending(r => r.CreatedAt)
|
||||
.FirstOrDefaultAsync(ct);
|
||||
|
||||
if (oauthReg is null)
|
||||
return "No OauthAppRegistry found — skipping OAuth2 cleanup.";
|
||||
|
||||
// Get Xibo client to delete the application
|
||||
var oauthSecret = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev));
|
||||
if (string.IsNullOrEmpty(oauthSecret))
|
||||
return $"OAuth secret not found for '{abbrev}' — cannot authenticate to delete app. Manual cleanup required.";
|
||||
|
||||
var xiboClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, oauthReg.ClientId, oauthSecret);
|
||||
await xiboClient.DeleteApplicationAsync(oauthReg.ClientId);
|
||||
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/decommission",
|
||||
Action = "oauth2-cleanup",
|
||||
Target = oauthReg.ClientId,
|
||||
Outcome = "success",
|
||||
Detail = $"OAuth2 application '{oauthReg.ClientId}' deleted from Xibo. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"OAuth2 application '{oauthReg.ClientId}' deleted.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: mysql-cleanup ───────────────────────────────────────────
|
||||
await runner.RunAsync("mysql-cleanup", async () =>
|
||||
{
|
||||
var mysqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
||||
var mysqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306");
|
||||
var mysqlAdminUser = await settings.GetAsync(SettingsService.MySqlAdminUser, "root");
|
||||
var mysqlAdminPassword = await settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
|
||||
|
||||
if (!int.TryParse(mysqlPort, out var port)) port = 3306;
|
||||
|
||||
var dbName = $"xibo_{abbrev}";
|
||||
var userName = $"xibo_{abbrev}";
|
||||
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
// DROP DATABASE
|
||||
var dropDbCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
|
||||
$"-p'{mysqlAdminPassword}' -e " +
|
||||
$"\"DROP DATABASE IF EXISTS \\`{dbName}\\`\"";
|
||||
RunSshCommand(sshClient, dropDbCmd);
|
||||
|
||||
// DROP USER
|
||||
var dropUserCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
|
||||
$"-p'{mysqlAdminPassword}' -e " +
|
||||
$"\"DROP USER IF EXISTS '{userName}'@'%'\"";
|
||||
RunSshCommand(sshClient, dropUserCmd);
|
||||
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/decommission",
|
||||
Action = "mysql-cleanup",
|
||||
Target = dbName,
|
||||
Outcome = "success",
|
||||
Detail = $"Database '{dbName}' and user '{userName}' dropped. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"Database '{dbName}' and user '{userName}' dropped on {mysqlHost}:{port}.";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 5: nfs-archive ─────────────────────────────────────────────
|
||||
await runner.RunAsync("nfs-archive", async () =>
|
||||
{
|
||||
var nfsServer = await settings.GetAsync(SettingsService.NfsServer);
|
||||
var nfsExport = await settings.GetAsync(SettingsService.NfsExport);
|
||||
var nfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(nfsServer))
|
||||
return "NFS server not configured — skipping archive.";
|
||||
|
||||
var export = (nfsExport ?? string.Empty).TrimEnd('/');
|
||||
var folder = (nfsExportFolder ?? string.Empty).Trim('/');
|
||||
var basePath = string.IsNullOrEmpty(folder) ? export : $"{export}/{folder}";
|
||||
|
||||
var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
|
||||
var sourcePath = $"{basePath}/{abbrev}";
|
||||
var archivePath = $"{basePath}/archived/{abbrev}-{timestamp}";
|
||||
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
// Temporarily mount NFS to move directories
|
||||
var mountPoint = $"/tmp/nfs-decommission-{abbrev}";
|
||||
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}");
|
||||
RunSshCommand(sshClient, $"sudo mount -t nfs4 {nfsServer}:{basePath} {mountPoint}");
|
||||
|
||||
try
|
||||
{
|
||||
// Ensure archive directory exists
|
||||
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}/archived");
|
||||
// Move — DO NOT delete (retain for 30 days minimum)
|
||||
RunSshCommand(sshClient, $"sudo mv {mountPoint}/{abbrev} {mountPoint}/archived/{abbrev}-{timestamp}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
RunSshCommandAllowFailure(sshClient, $"sudo umount {mountPoint}");
|
||||
RunSshCommandAllowFailure(sshClient, $"sudo rmdir {mountPoint}");
|
||||
}
|
||||
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/decommission",
|
||||
Action = "nfs-archive",
|
||||
Target = sourcePath,
|
||||
Outcome = "success",
|
||||
Detail = $"NFS data archived to {archivePath}. Retained for minimum 30 days. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"NFS data moved from {sourcePath} to {archivePath}. Retained for 30+ days.";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 6: registry-update ─────────────────────────────────────────
|
||||
await runner.RunAsync("registry-update", async () =>
|
||||
{
|
||||
var customer = await db.Customers.FirstOrDefaultAsync(c => c.Id == ctx.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {ctx.CustomerId} not found.");
|
||||
var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct)
|
||||
?? throw new InvalidOperationException($"Instance {ctx.InstanceId} not found.");
|
||||
|
||||
customer.Status = CustomerStatus.Decommissioned;
|
||||
instance.HealthStatus = HealthStatus.Critical;
|
||||
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/decommission",
|
||||
Action = "decommission-complete",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"Instance fully decommissioned. Customer status → Decommissioned. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
await hub.Clients.All.SendInstanceStatusChanged(
|
||||
ctx.CustomerId.ToString(), CustomerStatus.Decommissioned.ToString());
|
||||
|
||||
return $"Customer '{abbrev}' → Decommissioned. Instance health → Critical. Broadcast sent.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation("DecommissionPipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers (shared pattern from Phase1Pipeline)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(SettingsService settings)
|
||||
{
|
||||
var host = await settings.GetAsync("Ssh.SwarmHost")
|
||||
?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost).");
|
||||
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
|
||||
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
|
||||
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
|
||||
var password = await settings.GetAsync("Ssh.SwarmPassword");
|
||||
|
||||
if (!int.TryParse(portStr, out var port)) port = 22;
|
||||
|
||||
return new SshConnectionInfo(host, port, user, keyPath, password);
|
||||
}
|
||||
|
||||
private static SshClient CreateSshClient(SshConnectionInfo info)
|
||||
{
|
||||
var authMethods = new List<AuthenticationMethod>();
|
||||
|
||||
if (!string.IsNullOrEmpty(info.KeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
|
||||
|
||||
if (!string.IsNullOrEmpty(info.Password))
|
||||
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
|
||||
|
||||
if (authMethods.Count == 0)
|
||||
{
|
||||
var defaultKeyPath = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
|
||||
if (File.Exists(defaultKeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
|
||||
else
|
||||
throw new InvalidOperationException(
|
||||
$"No SSH authentication method available for {info.Host}:{info.Port}.");
|
||||
}
|
||||
|
||||
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
|
||||
return new SshClient(connInfo);
|
||||
}
|
||||
|
||||
private static string RunSshCommand(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
if (cmd.ExitStatus != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
|
||||
return cmd.Result;
|
||||
}
|
||||
|
||||
private static void RunSshCommandAllowFailure(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
// Intentionally ignore exit code — used for idempotent cleanup operations
|
||||
}
|
||||
|
||||
internal sealed record SshConnectionInfo(
|
||||
string Host, int Port, string Username, string? KeyPath, string? Password);
|
||||
}
|
||||
37
OTSSignsOrchestrator.Server/Workers/IProvisioningPipeline.cs
Normal file
37
OTSSignsOrchestrator.Server/Workers/IProvisioningPipeline.cs
Normal file
@@ -0,0 +1,37 @@
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Common interface for all provisioning pipelines (Phase1, Phase2, etc.).
|
||||
/// Resolved from DI via <see cref="IEnumerable{IProvisioningPipeline}"/> and matched by
|
||||
/// <see cref="HandlesJobType"/>.
|
||||
/// </summary>
|
||||
public interface IProvisioningPipeline
|
||||
{
|
||||
/// <summary>The <see cref="Job.JobType"/> this pipeline handles (e.g. "provision", "bootstrap").</summary>
|
||||
string HandlesJobType { get; }
|
||||
|
||||
/// <summary>Execute the pipeline steps for the given job.</summary>
|
||||
Task ExecuteAsync(Job job, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shared context extracted from <see cref="Job.Parameters"/> JSON and the associated
|
||||
/// <see cref="Customer"/> / <see cref="Instance"/> entities. Passed between pipeline steps.
|
||||
/// </summary>
|
||||
public sealed record PipelineContext
|
||||
{
|
||||
public required Guid JobId { get; init; }
|
||||
public required Guid CustomerId { get; init; }
|
||||
public required Guid InstanceId { get; init; }
|
||||
public required string Abbreviation { get; init; }
|
||||
public required string CompanyName { get; init; }
|
||||
public required string AdminEmail { get; init; }
|
||||
public required string AdminFirstName { get; init; }
|
||||
public required string InstanceUrl { get; init; }
|
||||
public required string DockerStackName { get; init; }
|
||||
|
||||
/// <summary>Raw parameters JSON from the Job entity for step-specific overrides.</summary>
|
||||
public string? ParametersJson { get; init; }
|
||||
}
|
||||
494
OTSSignsOrchestrator.Server/Workers/Phase1Pipeline.cs
Normal file
494
OTSSignsOrchestrator.Server/Workers/Phase1Pipeline.cs
Normal file
@@ -0,0 +1,494 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Renci.SshNet;
|
||||
using OTSSignsOrchestrator.Core.Configuration;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
using static OTSSignsOrchestrator.Core.Configuration.AppConstants;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Phase 1 provisioning pipeline — infrastructure setup. Handles <c>JobType = "provision"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. mysql-setup — Create DB + user on external MySQL via SSH tunnel
|
||||
/// 2. docker-secrets — Create Docker Swarm secrets via SSH
|
||||
/// 3. nfs-dirs — Create NFS sub-directories via SSH
|
||||
/// 4. authentik-provision — SAML provider, application, groups, invitation flow in Authentik
|
||||
/// 5. stack-deploy — Render compose YAML and deploy via <c>docker stack deploy</c>
|
||||
/// 6. credential-store — Store generated credentials in Bitwarden Secrets Manager
|
||||
/// </summary>
|
||||
public sealed class Phase1Pipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "provision";
|
||||
|
||||
private const int TotalSteps = 6;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<Phase1Pipeline> _logger;
|
||||
|
||||
public Phase1Pipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<Phase1Pipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var authentikClient = scope.ServiceProvider.GetRequiredService<IAuthentikClient>();
|
||||
var composeRenderer = scope.ServiceProvider.GetRequiredService<ComposeRenderService>();
|
||||
var gitService = scope.ServiceProvider.GetRequiredService<GitTemplateService>();
|
||||
var bws = scope.ServiceProvider.GetRequiredService<IBitwardenSecretService>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
var stackName = ctx.DockerStackName;
|
||||
|
||||
// Mutable state accumulated across steps
|
||||
var mysqlPassword = GenerateRandomPassword(32);
|
||||
var mysqlDbName = $"xibo_{abbrev}";
|
||||
var mysqlUserName = $"xibo_{abbrev}";
|
||||
string? authentikProviderId = null;
|
||||
|
||||
// ── Step 1: mysql-setup ─────────────────────────────────────────────
|
||||
await runner.RunAsync("mysql-setup", async () =>
|
||||
{
|
||||
var mysqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost");
|
||||
var mysqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306");
|
||||
var mysqlAdminUser = await settings.GetAsync(SettingsService.MySqlAdminUser, "root");
|
||||
var mysqlAdminPassword = await settings.GetAsync(SettingsService.MySqlAdminPassword, string.Empty);
|
||||
|
||||
if (!int.TryParse(mysqlPort, out var port)) port = 3306;
|
||||
|
||||
// SSH to MySQL host, create database and user
|
||||
// Reuse pattern from InstanceService.CreateMySqlDatabaseAsync — runs SQL
|
||||
// via SSH tunnel to the MySQL server.
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
// Create database
|
||||
var createDbCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
|
||||
$"-p'{mysqlAdminPassword}' -e " +
|
||||
$"\"CREATE DATABASE IF NOT EXISTS \\`{mysqlDbName}\\`\"";
|
||||
RunSshCommand(sshClient, createDbCmd);
|
||||
|
||||
// Create user
|
||||
var createUserCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
|
||||
$"-p'{mysqlAdminPassword}' -e " +
|
||||
$"\"CREATE USER IF NOT EXISTS '{mysqlUserName}'@'%' IDENTIFIED BY '{mysqlPassword}'\"";
|
||||
RunSshCommand(sshClient, createUserCmd);
|
||||
|
||||
// Grant privileges
|
||||
var grantCmd = $"mysql -h {mysqlHost} -P {port} -u {mysqlAdminUser} " +
|
||||
$"-p'{mysqlAdminPassword}' -e " +
|
||||
$"\"GRANT ALL PRIVILEGES ON \\`{mysqlDbName}\\`.* TO '{mysqlUserName}'@'%'; FLUSH PRIVILEGES;\"";
|
||||
RunSshCommand(sshClient, grantCmd);
|
||||
|
||||
return $"Database '{mysqlDbName}' and user '{mysqlUserName}' created on {mysqlHost}:{port}.";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: docker-secrets ──────────────────────────────────────────
|
||||
await runner.RunAsync("docker-secrets", async () =>
|
||||
{
|
||||
// Reuse IDockerSecretsService pattern — create secrets via SSH docker CLI
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
var secrets = new Dictionary<string, string>
|
||||
{
|
||||
[CustomerMysqlPasswordSecretName(abbrev)] = mysqlPassword,
|
||||
[CustomerMysqlUserSecretName(abbrev)] = mysqlUserName,
|
||||
[GlobalMysqlHostSecretName] = await settings.GetAsync(SettingsService.MySqlHost, "localhost"),
|
||||
[GlobalMysqlPortSecretName] = await settings.GetAsync(SettingsService.MySqlPort, "3306"),
|
||||
};
|
||||
|
||||
var created = new List<string>();
|
||||
foreach (var (name, value) in secrets)
|
||||
{
|
||||
// Remove existing secret if present (idempotent rotate)
|
||||
RunSshCommandAllowFailure(sshClient, $"docker secret rm {name}");
|
||||
|
||||
var safeValue = value.Replace("'", "'\\''");
|
||||
var cmd = $"printf '%s' '{safeValue}' | docker secret create {name} -";
|
||||
RunSshCommand(sshClient, cmd);
|
||||
created.Add(name);
|
||||
}
|
||||
|
||||
return $"Docker secrets created: {string.Join(", ", created)}.";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: nfs-dirs ────────────────────────────────────────────────
|
||||
await runner.RunAsync("nfs-dirs", async () =>
|
||||
{
|
||||
var nfsServer = await settings.GetAsync(SettingsService.NfsServer);
|
||||
var nfsExport = await settings.GetAsync(SettingsService.NfsExport);
|
||||
var nfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(nfsServer))
|
||||
return "NFS server not configured — skipping directory creation.";
|
||||
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
// Build the base path for NFS dirs
|
||||
var export = (nfsExport ?? string.Empty).TrimEnd('/');
|
||||
var folder = (nfsExportFolder ?? string.Empty).Trim('/');
|
||||
var basePath = string.IsNullOrEmpty(folder) ? export : $"{export}/{folder}";
|
||||
|
||||
var subdirs = new[]
|
||||
{
|
||||
$"{abbrev}/cms-custom",
|
||||
$"{abbrev}/cms-backup",
|
||||
$"{abbrev}/cms-library",
|
||||
$"{abbrev}/cms-userscripts",
|
||||
$"{abbrev}/cms-ca-certs",
|
||||
};
|
||||
|
||||
// Create directories via sudo on the NFS host
|
||||
// The swarm node mounts NFS, so we can create directories by
|
||||
// temporarily mounting, creating, then unmounting.
|
||||
var mountPoint = $"/tmp/nfs-provision-{abbrev}";
|
||||
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}");
|
||||
RunSshCommand(sshClient,
|
||||
$"sudo mount -t nfs4 {nfsServer}:{basePath} {mountPoint}");
|
||||
|
||||
try
|
||||
{
|
||||
foreach (var subdir in subdirs)
|
||||
{
|
||||
RunSshCommand(sshClient, $"sudo mkdir -p {mountPoint}/{subdir}");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
RunSshCommandAllowFailure(sshClient, $"sudo umount {mountPoint}");
|
||||
RunSshCommandAllowFailure(sshClient, $"sudo rmdir {mountPoint}");
|
||||
}
|
||||
|
||||
return $"NFS directories created under {basePath}: {string.Join(", ", subdirs)}.";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: authentik-provision ─────────────────────────────────────
|
||||
await runner.RunAsync("authentik-provision", async () =>
|
||||
{
|
||||
var instanceUrl = ctx.InstanceUrl;
|
||||
var authFlowSlug = await settings.GetAsync(SettingsService.AuthentikAuthorizationFlowSlug, "default-provider-authorization-implicit-consent");
|
||||
var signingKpId = await settings.GetAsync(SettingsService.AuthentikSigningKeypairId);
|
||||
|
||||
// 1. Create SAML provider
|
||||
var providerResponse = await authentikClient.CreateSamlProviderAsync(new CreateSamlProviderRequest(
|
||||
Name: $"xibo-{abbrev}",
|
||||
AuthorizationFlow: authFlowSlug,
|
||||
AcsUrl: $"{instanceUrl}/saml/acs",
|
||||
Issuer: $"xibo-{abbrev}",
|
||||
SpBinding: "post",
|
||||
Audience: instanceUrl,
|
||||
SigningKp: signingKpId));
|
||||
|
||||
EnsureSuccess(providerResponse);
|
||||
var providerData = providerResponse.Content
|
||||
?? throw new InvalidOperationException("Authentik SAML provider creation returned empty response.");
|
||||
authentikProviderId = providerData["pk"]?.ToString();
|
||||
|
||||
// 2. Create Authentik application linked to provider
|
||||
var appResponse = await authentikClient.CreateApplicationAsync(new CreateAuthentikApplicationRequest(
|
||||
Name: $"xibo-{abbrev}",
|
||||
Slug: $"xibo-{abbrev}",
|
||||
Provider: authentikProviderId!,
|
||||
MetaLaunchUrl: instanceUrl));
|
||||
EnsureSuccess(appResponse);
|
||||
|
||||
// 3. Create 4 tenant groups
|
||||
var groupNames = new[]
|
||||
{
|
||||
$"customer-{abbrev}",
|
||||
$"customer-{abbrev}-viewer",
|
||||
$"customer-{abbrev}-editor",
|
||||
$"customer-{abbrev}-admin",
|
||||
};
|
||||
|
||||
foreach (var groupName in groupNames)
|
||||
{
|
||||
var groupResp = await authentikClient.CreateGroupAsync(new CreateAuthentikGroupRequest(
|
||||
Name: groupName,
|
||||
IsSuperuser: false,
|
||||
Parent: null));
|
||||
EnsureSuccess(groupResp);
|
||||
}
|
||||
|
||||
// 4. Create invitation flow
|
||||
var inviteResponse = await authentikClient.CreateInvitationAsync(new CreateFlowRequest(
|
||||
Name: $"invite-{abbrev}",
|
||||
SingleUse: true,
|
||||
Expires: DateTimeOffset.UtcNow.AddDays(30)));
|
||||
EnsureSuccess(inviteResponse);
|
||||
|
||||
// Store Authentik provider ID on the Instance entity
|
||||
var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct);
|
||||
if (instance is not null)
|
||||
{
|
||||
instance.AuthentikProviderId = authentikProviderId;
|
||||
await db.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
return $"Authentik SAML provider '{authentikProviderId}', application 'xibo-{abbrev}', " +
|
||||
$"4 groups, and invitation flow created.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 5: stack-deploy ────────────────────────────────────────────
|
||||
await runner.RunAsync("stack-deploy", async () =>
|
||||
{
|
||||
// Fetch template
|
||||
var repoUrl = await settings.GetAsync(SettingsService.GitRepoUrl);
|
||||
var repoPat = await settings.GetAsync(SettingsService.GitRepoPat);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(repoUrl))
|
||||
throw new InvalidOperationException("Git template repository URL is not configured.");
|
||||
|
||||
var templateConfig = await gitService.FetchAsync(repoUrl, repoPat);
|
||||
|
||||
// Build render context
|
||||
var renderCtx = new RenderContext
|
||||
{
|
||||
CustomerName = ctx.CompanyName,
|
||||
CustomerAbbrev = abbrev,
|
||||
StackName = stackName,
|
||||
CmsServerName = await settings.GetAsync(SettingsService.DefaultCmsServerNameTemplate, "app.ots-signs.com"),
|
||||
HostHttpPort = 80,
|
||||
CmsImage = await settings.GetAsync(SettingsService.DefaultCmsImage, "ghcr.io/xibosignage/xibo-cms:release-4.2.3"),
|
||||
MemcachedImage = await settings.GetAsync(SettingsService.DefaultMemcachedImage, "memcached:alpine"),
|
||||
QuickChartImage = await settings.GetAsync(SettingsService.DefaultQuickChartImage, "ianw/quickchart"),
|
||||
NewtImage = await settings.GetAsync(SettingsService.DefaultNewtImage, "fosrl/newt"),
|
||||
ThemeHostPath = await settings.GetAsync(SettingsService.DefaultThemeHostPath, "/cms/ots-theme"),
|
||||
MySqlHost = await settings.GetAsync(SettingsService.MySqlHost, "localhost"),
|
||||
MySqlPort = await settings.GetAsync(SettingsService.MySqlPort, "3306"),
|
||||
MySqlDatabase = mysqlDbName,
|
||||
MySqlUser = mysqlUserName,
|
||||
MySqlPassword = mysqlPassword,
|
||||
SmtpServer = await settings.GetAsync(SettingsService.SmtpServer, string.Empty),
|
||||
SmtpUsername = await settings.GetAsync(SettingsService.SmtpUsername, string.Empty),
|
||||
SmtpPassword = await settings.GetAsync(SettingsService.SmtpPassword, string.Empty),
|
||||
SmtpUseTls = await settings.GetAsync(SettingsService.SmtpUseTls, "YES"),
|
||||
SmtpUseStartTls = await settings.GetAsync(SettingsService.SmtpUseStartTls, "YES"),
|
||||
SmtpRewriteDomain = await settings.GetAsync(SettingsService.SmtpRewriteDomain, string.Empty),
|
||||
SmtpHostname = await settings.GetAsync(SettingsService.SmtpHostname, string.Empty),
|
||||
SmtpFromLineOverride = await settings.GetAsync(SettingsService.SmtpFromLineOverride, "NO"),
|
||||
PhpPostMaxSize = await settings.GetAsync(SettingsService.DefaultPhpPostMaxSize, "10G"),
|
||||
PhpUploadMaxFilesize = await settings.GetAsync(SettingsService.DefaultPhpUploadMaxFilesize, "10G"),
|
||||
PhpMaxExecutionTime = await settings.GetAsync(SettingsService.DefaultPhpMaxExecutionTime, "600"),
|
||||
PangolinEndpoint = await settings.GetAsync(SettingsService.PangolinEndpoint, "https://app.pangolin.net"),
|
||||
NfsServer = await settings.GetAsync(SettingsService.NfsServer),
|
||||
NfsExport = await settings.GetAsync(SettingsService.NfsExport),
|
||||
NfsExportFolder = await settings.GetAsync(SettingsService.NfsExportFolder),
|
||||
NfsExtraOptions = await settings.GetAsync(SettingsService.NfsOptions, string.Empty),
|
||||
};
|
||||
|
||||
var composeYaml = composeRenderer.Render(templateConfig.Yaml, renderCtx);
|
||||
|
||||
// Deploy via SSH: pipe compose YAML to docker stack deploy
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
var safeYaml = composeYaml.Replace("'", "'\\''");
|
||||
var deployCmd = $"printf '%s' '{safeYaml}' | docker stack deploy -c - {stackName}";
|
||||
var result = RunSshCommand(sshClient, deployCmd);
|
||||
return $"Stack '{stackName}' deployed. Output: {result}";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 6: credential-store ────────────────────────────────────────
|
||||
await runner.RunAsync("credential-store", async () =>
|
||||
{
|
||||
// Bitwarden item structure:
|
||||
// Key: "{abbrev}/mysql-password" → MySQL password for xibo_{abbrev} user
|
||||
// Key: "{abbrev}/mysql-database" → Database name (for reference)
|
||||
// Key: "{abbrev}/mysql-user" → MySQL username (for reference)
|
||||
// All stored in the instance Bitwarden project via IBitwardenSecretService.
|
||||
|
||||
await bws.CreateInstanceSecretAsync(
|
||||
key: $"{abbrev}/mysql-password",
|
||||
value: mysqlPassword,
|
||||
note: $"MySQL password for instance {abbrev}. DB: {mysqlDbName}, User: {mysqlUserName}");
|
||||
|
||||
// Also persist in settings for Phase2 and future redeploys
|
||||
await settings.SetAsync(
|
||||
SettingsService.InstanceMySqlPassword(abbrev),
|
||||
mysqlPassword,
|
||||
SettingsService.CatInstance,
|
||||
isSensitive: true);
|
||||
|
||||
// Clear in-memory password
|
||||
mysqlPassword = string.Empty;
|
||||
|
||||
return $"Credentials stored in Bitwarden for instance '{abbrev}'.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation("Phase1Pipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads SSH connection details for the Docker Swarm host from settings.
|
||||
/// Expects settings keys: "Ssh.SwarmHost", "Ssh.SwarmPort", "Ssh.SwarmUser", "Ssh.SwarmKeyPath".
|
||||
/// </summary>
|
||||
private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(SettingsService settings)
|
||||
{
|
||||
var host = await settings.GetAsync("Ssh.SwarmHost")
|
||||
?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost).");
|
||||
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
|
||||
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
|
||||
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
|
||||
var password = await settings.GetAsync("Ssh.SwarmPassword");
|
||||
|
||||
if (!int.TryParse(portStr, out var port)) port = 22;
|
||||
|
||||
return new SshConnectionInfo(host, port, user, keyPath, password);
|
||||
}
|
||||
|
||||
private static SshClient CreateSshClient(SshConnectionInfo info)
|
||||
{
|
||||
var authMethods = new List<AuthenticationMethod>();
|
||||
|
||||
if (!string.IsNullOrEmpty(info.KeyPath))
|
||||
{
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(
|
||||
info.Username, new PrivateKeyFile(info.KeyPath)));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(info.Password))
|
||||
{
|
||||
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
|
||||
}
|
||||
|
||||
if (authMethods.Count == 0)
|
||||
{
|
||||
var defaultKeyPath = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
|
||||
if (File.Exists(defaultKeyPath))
|
||||
{
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(
|
||||
info.Username, new PrivateKeyFile(defaultKeyPath)));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"No SSH authentication method available for {info.Host}:{info.Port}. " +
|
||||
"Configure Ssh.SwarmKeyPath or Ssh.SwarmPassword in settings.");
|
||||
}
|
||||
}
|
||||
|
||||
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
|
||||
return new SshClient(connInfo);
|
||||
}
|
||||
|
||||
private static string RunSshCommand(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
if (cmd.ExitStatus != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
|
||||
return cmd.Result;
|
||||
}
|
||||
|
||||
private static void RunSshCommandAllowFailure(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
// Intentionally ignore exit code — used for idempotent cleanup operations
|
||||
}
|
||||
|
||||
private static string GenerateRandomPassword(int length)
|
||||
{
|
||||
const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*";
|
||||
return RandomNumberGenerator.GetString(chars, length);
|
||||
}
|
||||
|
||||
private static void EnsureSuccess<T>(Refit.IApiResponse<T> response)
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException(
|
||||
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
|
||||
}
|
||||
|
||||
/// <summary>SSH connection details read from Bitwarden settings.</summary>
|
||||
internal sealed record SshConnectionInfo(
|
||||
string Host,
|
||||
int Port,
|
||||
string Username,
|
||||
string? KeyPath,
|
||||
string? Password);
|
||||
}
|
||||
479
OTSSignsOrchestrator.Server/Workers/Phase2Pipeline.cs
Normal file
479
OTSSignsOrchestrator.Server/Workers/Phase2Pipeline.cs
Normal file
@@ -0,0 +1,479 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Quartz;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
using OTSSignsOrchestrator.Server.Services;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Phase 2 provisioning pipeline — Xibo CMS bootstrap. Handles <c>JobType = "bootstrap"</c>.
|
||||
/// Triggered after the stack deployed in Phase 1 becomes healthy.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. xibo-health-check — Poll GET /about until 200 OK
|
||||
/// 2. create-ots-admin — POST /api/user (superAdmin)
|
||||
/// 3. create-ots-svc — POST /api/user (service account)
|
||||
/// 4. create-oauth2-app — POST /api/application — IMMEDIATELY capture secret
|
||||
/// 5. create-groups — POST /api/group (viewer, editor, admin, ots-it)
|
||||
/// 6. assign-group-acl — POST /api/group/{id}/acl per role
|
||||
/// 7. assign-service-accounts — Assign admin + svc to ots-it group
|
||||
/// 8. apply-theme — PUT /api/settings THEME_NAME=otssigns
|
||||
/// 9. delete-default-user — Delete xibo_admin (after safety check)
|
||||
/// 10. schedule-snapshot — Register Quartz DailySnapshotJob
|
||||
/// 11. authentik-invite — Create Authentik user + invitation, send welcome email
|
||||
/// </summary>
|
||||
public sealed class Phase2Pipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "bootstrap";
|
||||
|
||||
private const int TotalSteps = 11;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<Phase2Pipeline> _logger;
|
||||
|
||||
public Phase2Pipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<Phase2Pipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var xiboFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
|
||||
var authentikClient = scope.ServiceProvider.GetRequiredService<IAuthentikClient>();
|
||||
var bws = scope.ServiceProvider.GetRequiredService<IBitwardenSecretService>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
var emailService = scope.ServiceProvider.GetRequiredService<EmailService>();
|
||||
var schedulerFactory = scope.ServiceProvider.GetRequiredService<ISchedulerFactory>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
var instanceUrl = ctx.InstanceUrl;
|
||||
|
||||
// The initial Xibo CMS ships with xibo_admin / password credentials.
|
||||
// We use these to bootstrap via the API.
|
||||
IXiboApiClient? xiboClient = null;
|
||||
|
||||
// Mutable state accumulated across steps
|
||||
var adminPassword = GenerateRandomPassword(24);
|
||||
var svcPassword = GenerateRandomPassword(24);
|
||||
int otsAdminUserId = 0;
|
||||
int otsSvcUserId = 0;
|
||||
string otsOAuthClientId = string.Empty;
|
||||
string otsOAuthClientSecret = string.Empty;
|
||||
var groupIds = new Dictionary<string, int>(); // groupName → groupId
|
||||
|
||||
// ── Step 1: xibo-health-check ───────────────────────────────────────
|
||||
await runner.RunAsync("xibo-health-check", async () =>
|
||||
{
|
||||
// Poll GET /about every 10 seconds, up to 5 minutes
|
||||
var timeout = TimeSpan.FromMinutes(5);
|
||||
var interval = TimeSpan.FromSeconds(10);
|
||||
var deadline = DateTimeOffset.UtcNow.Add(timeout);
|
||||
|
||||
// Create a temporary client using default bootstrap credentials
|
||||
// xibo_admin/password → OAuth client_credentials from the seed application
|
||||
// The first call is to /about which doesn't need auth, use a raw HttpClient
|
||||
using var httpClient = new HttpClient { BaseAddress = new Uri(instanceUrl.TrimEnd('/')) };
|
||||
|
||||
while (DateTimeOffset.UtcNow < deadline)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var response = await httpClient.GetAsync("/api/about", ct);
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
return $"Xibo CMS at {instanceUrl} is healthy (status {(int)response.StatusCode}).";
|
||||
}
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
// Not ready yet
|
||||
}
|
||||
|
||||
await Task.Delay(interval, ct);
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"Xibo CMS at {instanceUrl} did not return 200 OK from /about within {timeout.TotalMinutes} minutes.");
|
||||
}, ct);
|
||||
|
||||
// Get a Refit client using the seed OAuth app credentials (xibo_admin bootstrap)
|
||||
// Parameters JSON should contain bootstrapClientId + bootstrapClientSecret
|
||||
var parameters = !string.IsNullOrEmpty(ctx.ParametersJson)
|
||||
? JsonDocument.Parse(ctx.ParametersJson)
|
||||
: null;
|
||||
|
||||
var bootstrapClientId = parameters?.RootElement.TryGetProperty("bootstrapClientId", out var bcid) == true
|
||||
? bcid.GetString() ?? string.Empty
|
||||
: string.Empty;
|
||||
var bootstrapClientSecret = parameters?.RootElement.TryGetProperty("bootstrapClientSecret", out var bcs) == true
|
||||
? bcs.GetString() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
if (string.IsNullOrEmpty(bootstrapClientId) || string.IsNullOrEmpty(bootstrapClientSecret))
|
||||
throw new InvalidOperationException(
|
||||
"Bootstrap OAuth credentials (bootstrapClientId, bootstrapClientSecret) must be provided in Job.Parameters.");
|
||||
|
||||
xiboClient = await xiboFactory.CreateAsync(instanceUrl, bootstrapClientId, bootstrapClientSecret);
|
||||
|
||||
// ── Step 2: create-ots-admin ────────────────────────────────────────
|
||||
await runner.RunAsync("create-ots-admin", async () =>
|
||||
{
|
||||
var username = $"ots-admin-{abbrev}";
|
||||
var email = $"ots-admin-{abbrev}@ots-signs.com";
|
||||
|
||||
var response = await xiboClient.CreateUserAsync(new CreateUserRequest(
|
||||
UserName: username,
|
||||
Email: email,
|
||||
Password: adminPassword,
|
||||
UserTypeId: 1, // SuperAdmin
|
||||
HomePageId: 1));
|
||||
|
||||
EnsureSuccess(response);
|
||||
var data = response.Content
|
||||
?? throw new InvalidOperationException("CreateUser returned empty response.");
|
||||
|
||||
otsAdminUserId = Convert.ToInt32(data["userId"]);
|
||||
return $"Created user '{username}' (userId={otsAdminUserId}, type=SuperAdmin).";
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: create-ots-svc ──────────────────────────────────────────
|
||||
await runner.RunAsync("create-ots-svc", async () =>
|
||||
{
|
||||
var username = $"ots-svc-{abbrev}";
|
||||
var email = $"ots-svc-{abbrev}@ots-signs.com";
|
||||
|
||||
var response = await xiboClient.CreateUserAsync(new CreateUserRequest(
|
||||
UserName: username,
|
||||
Email: email,
|
||||
Password: svcPassword,
|
||||
UserTypeId: 1, // SuperAdmin — service account needs full API access
|
||||
HomePageId: 1));
|
||||
|
||||
EnsureSuccess(response);
|
||||
var data = response.Content
|
||||
?? throw new InvalidOperationException("CreateUser returned empty response.");
|
||||
|
||||
otsSvcUserId = Convert.ToInt32(data["userId"]);
|
||||
return $"Created user '{username}' (userId={otsSvcUserId}, type=SuperAdmin).";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: create-oauth2-app ───────────────────────────────────────
|
||||
// CRITICAL: POST /api/application — NOT GET (that's blocked).
|
||||
// The OAuth2 client secret is returned ONLY in this response. Capture immediately.
|
||||
await runner.RunAsync("create-oauth2-app", async () =>
|
||||
{
|
||||
var appName = $"OTS Orchestrator — {abbrev.ToUpperInvariant()}";
|
||||
var response = await xiboClient.CreateApplicationAsync(new CreateApplicationRequest(appName));
|
||||
|
||||
EnsureSuccess(response);
|
||||
var data = response.Content
|
||||
?? throw new InvalidOperationException("CreateApplication returned empty response.");
|
||||
|
||||
// CRITICAL: Capture secret immediately — it cannot be retrieved again.
|
||||
otsOAuthClientId = data["key"]?.ToString()
|
||||
?? throw new InvalidOperationException("OAuth application response missing 'key'.");
|
||||
otsOAuthClientSecret = data["secret"]?.ToString()
|
||||
?? throw new InvalidOperationException("OAuth application response missing 'secret'.");
|
||||
|
||||
// Store to Bitwarden CLI wrapper
|
||||
await bws.CreateInstanceSecretAsync(
|
||||
key: $"{abbrev}/xibo-oauth-secret",
|
||||
value: otsOAuthClientSecret,
|
||||
note: $"Xibo CMS OAuth2 client secret for OTS application. ClientId: {otsOAuthClientId}");
|
||||
|
||||
// Store clientId ONLY in the database — NEVER store the secret in the DB
|
||||
db.OauthAppRegistries.Add(new OauthAppRegistry
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
ClientId = otsOAuthClientId,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"OAuth2 app '{appName}' created. ClientId={otsOAuthClientId}. Secret stored in Bitwarden (NEVER in DB).";
|
||||
}, ct);
|
||||
|
||||
// ── Step 5: create-groups ───────────────────────────────────────────
|
||||
await runner.RunAsync("create-groups", async () =>
|
||||
{
|
||||
var groupDefs = new[]
|
||||
{
|
||||
($"{abbrev}-viewer", "Viewer role"),
|
||||
($"{abbrev}-editor", "Editor role"),
|
||||
($"{abbrev}-admin", "Admin role"),
|
||||
($"ots-it-{abbrev}", "OTS IT internal"),
|
||||
};
|
||||
|
||||
foreach (var (name, description) in groupDefs)
|
||||
{
|
||||
var response = await xiboClient.CreateGroupAsync(new CreateGroupRequest(name, description));
|
||||
EnsureSuccess(response);
|
||||
var data = response.Content
|
||||
?? throw new InvalidOperationException($"CreateGroup '{name}' returned empty response.");
|
||||
groupIds[name] = Convert.ToInt32(data["groupId"]);
|
||||
}
|
||||
|
||||
return $"Created {groupDefs.Length} groups: {string.Join(", ", groupIds.Keys)}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 6: assign-group-acl ────────────────────────────────────────
|
||||
// POST /api/group/{id}/acl — NOT /features
|
||||
await runner.RunAsync("assign-group-acl", async () =>
|
||||
{
|
||||
var aclMap = new Dictionary<string, (string[] ObjectIds, string[] PermissionIds)>
|
||||
{
|
||||
[$"{abbrev}-viewer"] = (XiboFeatureManifests.ViewerObjectIds, XiboFeatureManifests.ViewerPermissionIds),
|
||||
[$"{abbrev}-editor"] = (XiboFeatureManifests.EditorObjectIds, XiboFeatureManifests.EditorPermissionIds),
|
||||
[$"{abbrev}-admin"] = (XiboFeatureManifests.AdminObjectIds, XiboFeatureManifests.AdminPermissionIds),
|
||||
[$"ots-it-{abbrev}"] = (XiboFeatureManifests.OtsItObjectIds, XiboFeatureManifests.OtsItPermissionIds),
|
||||
};
|
||||
|
||||
var applied = new List<string>();
|
||||
foreach (var (groupName, (objectIds, permissionIds)) in aclMap)
|
||||
{
|
||||
if (!groupIds.TryGetValue(groupName, out var groupId))
|
||||
throw new InvalidOperationException($"Group '{groupName}' ID not found.");
|
||||
|
||||
var response = await xiboClient.SetGroupAclAsync(groupId, new SetAclRequest(objectIds, permissionIds));
|
||||
EnsureSuccess(response);
|
||||
applied.Add($"{groupName} ({objectIds.Length} features)");
|
||||
}
|
||||
|
||||
return $"ACL assigned: {string.Join(", ", applied)}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 7: assign-service-accounts ─────────────────────────────────
|
||||
await runner.RunAsync("assign-service-accounts", async () =>
|
||||
{
|
||||
var otsItGroupName = $"ots-it-{abbrev}";
|
||||
if (!groupIds.TryGetValue(otsItGroupName, out var otsItGroupId))
|
||||
throw new InvalidOperationException($"Group '{otsItGroupName}' ID not found.");
|
||||
|
||||
// Assign both ots-admin and ots-svc to the ots-it group
|
||||
var response = await xiboClient.AssignUserToGroupAsync(
|
||||
otsItGroupId,
|
||||
new AssignMemberRequest([otsAdminUserId, otsSvcUserId]));
|
||||
EnsureSuccess(response);
|
||||
|
||||
return $"Assigned ots-admin-{abbrev} (id={otsAdminUserId}) and ots-svc-{abbrev} (id={otsSvcUserId}) to group '{otsItGroupName}'.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 8: apply-theme ─────────────────────────────────────────────
|
||||
await runner.RunAsync("apply-theme", async () =>
|
||||
{
|
||||
var response = await xiboClient.UpdateSettingsAsync(new UpdateSettingsRequest(
|
||||
new Dictionary<string, string> { ["THEME_NAME"] = "otssigns" }));
|
||||
EnsureSuccess(response);
|
||||
|
||||
return "Theme set to 'otssigns'.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 9: delete-default-user ─────────────────────────────────────
|
||||
await runner.RunAsync("delete-default-user", async () =>
|
||||
{
|
||||
// First verify ots-admin works via OAuth2 client_credentials test
|
||||
var testClient = await xiboFactory.CreateAsync(instanceUrl, otsOAuthClientId, otsOAuthClientSecret);
|
||||
var aboutResponse = await testClient.GetAboutAsync();
|
||||
if (!aboutResponse.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException(
|
||||
$"OAuth2 verification failed for ots-admin app (status {aboutResponse.StatusCode}). " +
|
||||
"Refusing to delete xibo_admin — it may be the only working admin account.");
|
||||
|
||||
// Use GetAllPagesAsync to find xibo_admin user — default page size is only 10
|
||||
var allUsers = await xiboClient.GetAllPagesAsync(
|
||||
(start, length) => xiboClient.GetUsersAsync(start, length));
|
||||
|
||||
var xiboAdminUser = allUsers.FirstOrDefault(u =>
|
||||
u.TryGetValue("userName", out var name) &&
|
||||
string.Equals(name?.ToString(), "xibo_admin", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (xiboAdminUser is null)
|
||||
return "xibo_admin user not found — may have already been deleted.";
|
||||
|
||||
var xiboAdminId = Convert.ToInt32(xiboAdminUser["userId"]);
|
||||
|
||||
// Safety check: never delete if it would be the last SuperAdmin
|
||||
var superAdminCount = allUsers.Count(u =>
|
||||
u.TryGetValue("userTypeId", out var typeId) &&
|
||||
Convert.ToInt32(typeId) == 1);
|
||||
|
||||
if (superAdminCount <= 1)
|
||||
throw new InvalidOperationException(
|
||||
"Cannot delete xibo_admin — it is the only SuperAdmin. " +
|
||||
"Ensure ots-admin was created as SuperAdmin before retrying.");
|
||||
|
||||
await xiboClient.DeleteUserAsync(xiboAdminId);
|
||||
|
||||
return $"Deleted xibo_admin (userId={xiboAdminId}). Remaining SuperAdmins: {superAdminCount - 1}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 10: schedule-snapshot ──────────────────────────────────────
|
||||
await runner.RunAsync("schedule-snapshot", async () =>
|
||||
{
|
||||
var scheduler = await schedulerFactory.GetScheduler(ct);
|
||||
|
||||
var jobKey = new JobKey($"snapshot-{abbrev}", "daily-snapshots");
|
||||
var triggerKey = new TriggerKey($"snapshot-trigger-{abbrev}", "daily-snapshots");
|
||||
|
||||
var quartzJob = JobBuilder.Create<DailySnapshotJob>()
|
||||
.WithIdentity(jobKey)
|
||||
.UsingJobData("abbrev", abbrev)
|
||||
.UsingJobData("instanceId", ctx.InstanceId.ToString())
|
||||
.StoreDurably()
|
||||
.Build();
|
||||
|
||||
var trigger = TriggerBuilder.Create()
|
||||
.WithIdentity(triggerKey)
|
||||
.WithCronSchedule("0 0 2 * * ?") // 2:00 AM daily
|
||||
.Build();
|
||||
|
||||
await scheduler.ScheduleJob(quartzJob, trigger, ct);
|
||||
|
||||
return $"DailySnapshotJob scheduled for instance '{abbrev}' — daily at 02:00 UTC.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 11: authentik-invite ───────────────────────────────────────
|
||||
await runner.RunAsync("authentik-invite", async () =>
|
||||
{
|
||||
var adminEmail = ctx.AdminEmail;
|
||||
var firstName = ctx.AdminFirstName;
|
||||
|
||||
// Create Authentik user
|
||||
var userResponse = await authentikClient.CreateUserAsync(new CreateAuthentikUserRequest(
|
||||
Username: adminEmail,
|
||||
Name: $"{firstName}",
|
||||
Email: adminEmail,
|
||||
Groups: [$"customer-{abbrev}", $"customer-{abbrev}-viewer"]));
|
||||
EnsureSuccess(userResponse);
|
||||
|
||||
// Create invitation with 7-day expiry
|
||||
var inviteResponse = await authentikClient.CreateInvitationAsync(new CreateFlowRequest(
|
||||
Name: $"invite-{abbrev}",
|
||||
SingleUse: true,
|
||||
Expires: DateTimeOffset.UtcNow.AddDays(7)));
|
||||
EnsureSuccess(inviteResponse);
|
||||
|
||||
var inviteData = inviteResponse.Content;
|
||||
var invitationLink = inviteData?.TryGetValue("pk", out var pk) == true
|
||||
? $"{instanceUrl}/if/flow/invite-{abbrev}/?itoken={pk}"
|
||||
: "(invitation link unavailable)";
|
||||
|
||||
// Send welcome email
|
||||
await emailService.SendWelcomeEmailAsync(adminEmail, firstName, instanceUrl, invitationLink);
|
||||
|
||||
return $"Authentik user '{adminEmail}' created, assigned to customer-{abbrev} + customer-{abbrev}-viewer. " +
|
||||
$"Invitation link: {invitationLink}. Welcome email sent.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation("Phase2Pipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static string GenerateRandomPassword(int length)
|
||||
{
|
||||
const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*";
|
||||
return RandomNumberGenerator.GetString(chars, length);
|
||||
}
|
||||
|
||||
private static void EnsureSuccess<T>(Refit.IApiResponse<T> response)
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException(
|
||||
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Quartz job placeholder for daily instance snapshots.
|
||||
/// Registered per-instance by Phase2Pipeline step 10.
|
||||
/// </summary>
|
||||
[DisallowConcurrentExecution]
|
||||
public sealed class DailySnapshotJob : IJob
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<DailySnapshotJob> _logger;
|
||||
|
||||
public DailySnapshotJob(IServiceProvider services, ILogger<DailySnapshotJob> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Execute(IJobExecutionContext context)
|
||||
{
|
||||
var abbrev = context.MergedJobDataMap.GetString("abbrev");
|
||||
var instanceIdStr = context.MergedJobDataMap.GetString("instanceId");
|
||||
|
||||
_logger.LogInformation("DailySnapshotJob running for instance {Abbrev} (id={InstanceId})",
|
||||
abbrev, instanceIdStr);
|
||||
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
|
||||
if (!Guid.TryParse(instanceIdStr, out var instanceId))
|
||||
{
|
||||
_logger.LogError("DailySnapshotJob: invalid instanceId '{InstanceId}'", instanceIdStr);
|
||||
return;
|
||||
}
|
||||
|
||||
db.ScreenSnapshots.Add(new ScreenSnapshot
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = instanceId,
|
||||
SnapshotDate = DateOnly.FromDateTime(DateTime.UtcNow),
|
||||
ScreenCount = 0,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
_logger.LogInformation("DailySnapshotJob completed for instance {Abbrev}", abbrev);
|
||||
}
|
||||
}
|
||||
127
OTSSignsOrchestrator.Server/Workers/ProvisioningWorker.cs
Normal file
127
OTSSignsOrchestrator.Server/Workers/ProvisioningWorker.cs
Normal file
@@ -0,0 +1,127 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Background service that polls <see cref="OrchestratorDbContext.Jobs"/> for queued work,
|
||||
/// claims one job at a time, resolves the correct <see cref="IProvisioningPipeline"/>,
|
||||
/// and delegates execution. All transitions are logged and broadcast via SignalR.
|
||||
/// </summary>
|
||||
public sealed class ProvisioningWorker : BackgroundService
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<ProvisioningWorker> _logger;
|
||||
|
||||
private static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(5);
|
||||
|
||||
public ProvisioningWorker(
|
||||
IServiceProvider services,
|
||||
ILogger<ProvisioningWorker> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("ProvisioningWorker started — polling every {Interval}s", PollInterval.TotalSeconds);
|
||||
|
||||
using var timer = new PeriodicTimer(PollInterval);
|
||||
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken))
|
||||
{
|
||||
try
|
||||
{
|
||||
await TryProcessNextJobAsync(stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "ProvisioningWorker: unhandled error during poll cycle");
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("ProvisioningWorker stopped");
|
||||
}
|
||||
|
||||
private async Task TryProcessNextJobAsync(CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
|
||||
// Atomically claim the oldest queued job
|
||||
var job = await db.Jobs
|
||||
.Where(j => j.Status == JobStatus.Queued)
|
||||
.OrderBy(j => j.CreatedAt)
|
||||
.FirstOrDefaultAsync(ct);
|
||||
|
||||
if (job is null)
|
||||
return;
|
||||
|
||||
// Optimistic concurrency: set Running + StartedAt
|
||||
job.Status = JobStatus.Running;
|
||||
job.StartedAt = DateTime.UtcNow;
|
||||
|
||||
try
|
||||
{
|
||||
await db.SaveChangesAsync(ct);
|
||||
}
|
||||
catch (DbUpdateConcurrencyException)
|
||||
{
|
||||
// Another worker already claimed this job
|
||||
_logger.LogDebug("Job {JobId} was claimed by another worker", job.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Job {JobId} claimed (type={JobType}, customer={CustomerId})",
|
||||
job.Id, job.JobType, job.CustomerId);
|
||||
|
||||
// Resolve the correct pipeline for this job type
|
||||
var pipelines = scope.ServiceProvider.GetRequiredService<IEnumerable<IProvisioningPipeline>>();
|
||||
var pipeline = pipelines.FirstOrDefault(p =>
|
||||
string.Equals(p.HandlesJobType, job.JobType, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (pipeline is null)
|
||||
{
|
||||
_logger.LogError("No pipeline registered for job type '{JobType}' (job {JobId})", job.JobType, job.Id);
|
||||
job.Status = JobStatus.Failed;
|
||||
job.ErrorMessage = $"No pipeline registered for job type '{job.JobType}'.";
|
||||
job.CompletedAt = DateTime.UtcNow;
|
||||
await db.SaveChangesAsync(ct);
|
||||
await hub.Clients.All.SendJobCompleted(job.Id.ToString(), false, job.ErrorMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await pipeline.ExecuteAsync(job, ct);
|
||||
|
||||
job.Status = JobStatus.Completed;
|
||||
job.CompletedAt = DateTime.UtcNow;
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
var summary = $"Job {job.JobType} completed for customer {job.CustomerId}.";
|
||||
_logger.LogInformation("Job {JobId} completed successfully", job.Id);
|
||||
await hub.Clients.All.SendJobCompleted(job.Id.ToString(), true, summary);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Job {JobId} failed: {Message}", job.Id, ex.Message);
|
||||
|
||||
job.Status = JobStatus.Failed;
|
||||
job.ErrorMessage = ex.Message;
|
||||
job.CompletedAt = DateTime.UtcNow;
|
||||
await db.SaveChangesAsync(CancellationToken.None);
|
||||
|
||||
await hub.Clients.All.SendJobCompleted(job.Id.ToString(), false, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
229
OTSSignsOrchestrator.Server/Workers/ReactivatePipeline.cs
Normal file
229
OTSSignsOrchestrator.Server/Workers/ReactivatePipeline.cs
Normal file
@@ -0,0 +1,229 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Renci.SshNet;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Subscription reactivation pipeline — scales up Docker services, verifies health, resets
|
||||
/// payment failure counters. Handles <c>JobType = "reactivate"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. scale-up — SSH docker service scale web=1, xmr=1
|
||||
/// 2. health-verify — Poll GET /about every 10s up to 3 minutes
|
||||
/// 3. update-status — Customer.Status = Active, reset FailedPaymentCount + FirstPaymentFailedAt
|
||||
/// 4. audit-log — Append-only AuditLog entry
|
||||
/// 5. broadcast — InstanceStatusChanged via FleetHub
|
||||
/// </summary>
|
||||
public sealed class ReactivatePipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "reactivate";
|
||||
|
||||
private const int TotalSteps = 5;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<ReactivatePipeline> _logger;
|
||||
|
||||
public ReactivatePipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<ReactivatePipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
|
||||
// ── Step 1: scale-up ────────────────────────────────────────────────
|
||||
await runner.RunAsync("scale-up", async () =>
|
||||
{
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
var cmd = $"docker service scale xibo-{abbrev}_web=1 xibo-{abbrev}_xmr=1";
|
||||
var result = RunSshCommand(sshClient, cmd);
|
||||
return $"Scaled up services for xibo-{abbrev}. Output: {result}";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: health-verify ───────────────────────────────────────────
|
||||
await runner.RunAsync("health-verify", async () =>
|
||||
{
|
||||
var timeout = TimeSpan.FromMinutes(3);
|
||||
var interval = TimeSpan.FromSeconds(10);
|
||||
var deadline = DateTimeOffset.UtcNow.Add(timeout);
|
||||
|
||||
using var httpClient = new HttpClient { BaseAddress = new Uri(ctx.InstanceUrl.TrimEnd('/')) };
|
||||
|
||||
while (DateTimeOffset.UtcNow < deadline)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var response = await httpClient.GetAsync("/api/about", ct);
|
||||
if (response.IsSuccessStatusCode)
|
||||
return $"Xibo CMS at {ctx.InstanceUrl} is healthy (status {(int)response.StatusCode}).";
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
// Not ready yet — keep polling
|
||||
}
|
||||
|
||||
await Task.Delay(interval, ct);
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"Xibo CMS at {ctx.InstanceUrl} did not return 200 OK from /about within {timeout.TotalMinutes} minutes.");
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: update-status ───────────────────────────────────────────
|
||||
await runner.RunAsync("update-status", async () =>
|
||||
{
|
||||
var customer = await db.Customers.FirstOrDefaultAsync(c => c.Id == ctx.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {ctx.CustomerId} not found.");
|
||||
var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct)
|
||||
?? throw new InvalidOperationException($"Instance {ctx.InstanceId} not found.");
|
||||
|
||||
customer.Status = CustomerStatus.Active;
|
||||
customer.FailedPaymentCount = 0;
|
||||
customer.FirstPaymentFailedAt = null;
|
||||
instance.HealthStatus = HealthStatus.Healthy;
|
||||
instance.LastHealthCheck = DateTime.UtcNow;
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"Customer '{abbrev}' status → Active, FailedPaymentCount → 0, instance health → Healthy.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: audit-log ───────────────────────────────────────────────
|
||||
await runner.RunAsync("audit-log", async () =>
|
||||
{
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/stripe-webhook",
|
||||
Action = "reactivate",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"Subscription reactivated. Services scaled to 1. Health verified. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return "AuditLog entry written for reactivation.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 5: broadcast ───────────────────────────────────────────────
|
||||
await runner.RunAsync("broadcast", async () =>
|
||||
{
|
||||
await hub.Clients.All.SendInstanceStatusChanged(
|
||||
ctx.CustomerId.ToString(), CustomerStatus.Active.ToString());
|
||||
return "Broadcast InstanceStatusChanged → Active.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation("ReactivatePipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers (shared pattern from Phase1Pipeline)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(SettingsService settings)
|
||||
{
|
||||
var host = await settings.GetAsync("Ssh.SwarmHost")
|
||||
?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost).");
|
||||
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
|
||||
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
|
||||
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
|
||||
var password = await settings.GetAsync("Ssh.SwarmPassword");
|
||||
|
||||
if (!int.TryParse(portStr, out var port)) port = 22;
|
||||
|
||||
return new SshConnectionInfo(host, port, user, keyPath, password);
|
||||
}
|
||||
|
||||
private static SshClient CreateSshClient(SshConnectionInfo info)
|
||||
{
|
||||
var authMethods = new List<AuthenticationMethod>();
|
||||
|
||||
if (!string.IsNullOrEmpty(info.KeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
|
||||
|
||||
if (!string.IsNullOrEmpty(info.Password))
|
||||
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
|
||||
|
||||
if (authMethods.Count == 0)
|
||||
{
|
||||
var defaultKeyPath = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
|
||||
if (File.Exists(defaultKeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
|
||||
else
|
||||
throw new InvalidOperationException(
|
||||
$"No SSH authentication method available for {info.Host}:{info.Port}.");
|
||||
}
|
||||
|
||||
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
|
||||
return new SshClient(connInfo);
|
||||
}
|
||||
|
||||
private static string RunSshCommand(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
if (cmd.ExitStatus != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
|
||||
return cmd.Result;
|
||||
}
|
||||
|
||||
internal sealed record SshConnectionInfo(
|
||||
string Host, int Port, string Username, string? KeyPath, string? Password);
|
||||
}
|
||||
274
OTSSignsOrchestrator.Server/Workers/RotateCredentialsPipeline.cs
Normal file
274
OTSSignsOrchestrator.Server/Workers/RotateCredentialsPipeline.cs
Normal file
@@ -0,0 +1,274 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// OAuth2 credential rotation pipeline — deletes the old Xibo OAuth app, creates a new one,
|
||||
/// stores the new credentials, and verifies access. Handles <c>JobType = "rotate-oauth2"</c>.
|
||||
///
|
||||
/// CRITICAL: OAuth2 clientId CHANGES on rotation — there is no in-place secret refresh.
|
||||
/// The secret is returned ONLY in the <c>POST /api/application</c> response.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. delete-old-app — Delete current OAuth2 application via Xibo API
|
||||
/// 2. create-new-app — POST /api/application — IMMEDIATELY capture secret
|
||||
/// 3. store-credentials — Update Bitwarden + OauthAppRegistry with new clientId
|
||||
/// 4. verify-access — Test new credentials via client_credentials flow
|
||||
/// 5. audit-log — Write AuditLog (or CRITICAL error with emergency creds if steps 3-4 fail)
|
||||
/// </summary>
|
||||
public sealed class RotateCredentialsPipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "rotate-oauth2";
|
||||
|
||||
private const int TotalSteps = 5;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<RotateCredentialsPipeline> _logger;
|
||||
|
||||
public RotateCredentialsPipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<RotateCredentialsPipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var xiboFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
|
||||
var bws = scope.ServiceProvider.GetRequiredService<IBitwardenSecretService>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
|
||||
// Load current OAuth2 credentials
|
||||
var currentReg = await db.OauthAppRegistries
|
||||
.Where(r => r.InstanceId == ctx.InstanceId)
|
||||
.OrderByDescending(r => r.CreatedAt)
|
||||
.FirstOrDefaultAsync(ct)
|
||||
?? throw new InvalidOperationException(
|
||||
$"No OauthAppRegistry found for instance {ctx.InstanceId}. Cannot rotate.");
|
||||
|
||||
var currentSecret = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev))
|
||||
?? throw new InvalidOperationException(
|
||||
$"OAuth secret not found in settings for instance '{abbrev}'. Cannot rotate.");
|
||||
|
||||
var currentClientId = currentReg.ClientId;
|
||||
|
||||
// Get a Xibo API client using the current (about-to-be-deleted) credentials
|
||||
var xiboClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, currentClientId, currentSecret);
|
||||
|
||||
// Mutable state — captured across steps
|
||||
string newClientId = string.Empty;
|
||||
string newSecret = string.Empty;
|
||||
|
||||
// ── Step 1: delete-old-app ──────────────────────────────────────────
|
||||
await runner.RunAsync("delete-old-app", async () =>
|
||||
{
|
||||
await xiboClient.DeleteApplicationAsync(currentClientId);
|
||||
return $"Old OAuth2 application deleted. ClientId={currentClientId}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: create-new-app ──────────────────────────────────────────
|
||||
// CRITICAL: After this step, the old credentials are gone. If subsequent steps fail,
|
||||
// the new clientId + secret MUST be logged for manual recovery.
|
||||
await runner.RunAsync("create-new-app", async () =>
|
||||
{
|
||||
// Need a client with some auth to create the new app. Use the instance's bootstrap
|
||||
// admin credentials (ots-admin user) via direct login if available, or we can
|
||||
// re-authenticate using the base URL. Since the old app is deleted, we need a
|
||||
// different auth mechanism. The Xibo CMS allows creating apps as any authenticated
|
||||
// super-admin. Use the ots-admin password from Bitwarden.
|
||||
//
|
||||
// However, the simplest path: the DELETE above invalidated the old token, but the
|
||||
// Xibo CMS still has the ots-admin and ots-svc users. We stored the admin password
|
||||
// in Bitwarden. Retrieve it and create a new factory client.
|
||||
var adminPassBwsId = await settings.GetAsync(SettingsService.InstanceAdminPasswordSecretId(abbrev));
|
||||
if (string.IsNullOrEmpty(adminPassBwsId))
|
||||
throw new InvalidOperationException(
|
||||
$"Admin password Bitwarden secret ID not found for '{abbrev}'. Cannot create new OAuth app.");
|
||||
|
||||
var adminPassSecret = await bws.GetSecretAsync(adminPassBwsId);
|
||||
var bootstrapClientId = await settings.GetAsync(SettingsService.XiboBootstrapClientId)
|
||||
?? throw new InvalidOperationException("Bootstrap OAuth client ID not configured.");
|
||||
var bootstrapClientSecret = await settings.GetAsync(SettingsService.XiboBootstrapClientSecret)
|
||||
?? throw new InvalidOperationException("Bootstrap OAuth client secret not configured.");
|
||||
|
||||
// Re-create client using bootstrap credentials
|
||||
var bootstrapClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, bootstrapClientId, bootstrapClientSecret);
|
||||
|
||||
var appName = $"OTS Orchestrator — {abbrev.ToUpperInvariant()}";
|
||||
var response = await bootstrapClient.CreateApplicationAsync(new CreateApplicationRequest(appName));
|
||||
|
||||
EnsureSuccess(response);
|
||||
var data = response.Content
|
||||
?? throw new InvalidOperationException("CreateApplication returned empty response.");
|
||||
|
||||
// CRITICAL: Capture secret IMMEDIATELY — it is returned ONLY in this response.
|
||||
newClientId = data["key"]?.ToString()
|
||||
?? throw new InvalidOperationException("OAuth application response missing 'key'.");
|
||||
newSecret = data["secret"]?.ToString()
|
||||
?? throw new InvalidOperationException("OAuth application response missing 'secret'.");
|
||||
|
||||
return $"New OAuth2 application created. ClientId={newClientId}. Secret captured (stored in next step).";
|
||||
}, ct);
|
||||
|
||||
// Steps 3-5: if ANY of these fail after step 2, we must log the credentials for recovery.
|
||||
// The old app is already deleted — there is no rollback path.
|
||||
try
|
||||
{
|
||||
// ── Step 3: store-credentials ───────────────────────────────────
|
||||
await runner.RunAsync("store-credentials", async () =>
|
||||
{
|
||||
// Update Bitwarden with new secret
|
||||
var existingBwsId = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev));
|
||||
if (!string.IsNullOrEmpty(existingBwsId))
|
||||
{
|
||||
await bws.UpdateSecretAsync(
|
||||
existingBwsId,
|
||||
$"{abbrev}/xibo-oauth-secret",
|
||||
newSecret,
|
||||
$"Xibo CMS OAuth2 client secret (rotated). ClientId: {newClientId}");
|
||||
}
|
||||
else
|
||||
{
|
||||
await bws.CreateInstanceSecretAsync(
|
||||
key: $"{abbrev}/xibo-oauth-secret",
|
||||
value: newSecret,
|
||||
note: $"Xibo CMS OAuth2 client secret (rotated). ClientId: {newClientId}");
|
||||
}
|
||||
|
||||
// Update OauthAppRegistry with new clientId
|
||||
db.OauthAppRegistries.Add(new OauthAppRegistry
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
ClientId = newClientId,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
// Also update the settings cache with the new client ID
|
||||
await settings.SetAsync(
|
||||
SettingsService.InstanceOAuthClientId(abbrev),
|
||||
newClientId,
|
||||
SettingsService.CatInstance,
|
||||
isSensitive: false);
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
return $"Credentials stored. ClientId={newClientId} in OauthAppRegistry + Bitwarden.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: verify-access ───────────────────────────────────────
|
||||
await runner.RunAsync("verify-access", async () =>
|
||||
{
|
||||
var verifyClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, newClientId, newSecret);
|
||||
var aboutResp = await verifyClient.GetAboutAsync();
|
||||
EnsureSuccess(aboutResp);
|
||||
return $"New credentials verified. GET /about succeeded on {ctx.InstanceUrl}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 5: audit-log ───────────────────────────────────────────
|
||||
await runner.RunAsync("audit-log", async () =>
|
||||
{
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/credential-rotation",
|
||||
Action = "rotate-oauth2",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"OAuth2 credentials rotated. Old clientId={currentClientId} → new clientId={newClientId}. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"AuditLog entry written. OAuth2 rotation complete.";
|
||||
}, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// CRITICAL: Steps 3-5 failed AFTER step 2 created the new app.
|
||||
// The old credentials are already deleted. Log new credentials for manual recovery.
|
||||
_logger.LogCritical(ex,
|
||||
"CRITICAL — OAuth2 rotation for '{Abbrev}' failed after new app creation. " +
|
||||
"Old clientId={OldClientId} (DELETED). " +
|
||||
"New clientId={NewClientId}, New secret={NewSecret}. " +
|
||||
"EMERGENCY RECOVERY DATA — store these credentials manually.",
|
||||
abbrev, currentClientId, newClientId, newSecret);
|
||||
|
||||
// Also persist to a JobStep for operator visibility
|
||||
var emergencyStep = new JobStep
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
JobId = job.Id,
|
||||
StepName = "emergency-credential-log",
|
||||
Status = JobStepStatus.Failed,
|
||||
StartedAt = DateTime.UtcNow,
|
||||
CompletedAt = DateTime.UtcNow,
|
||||
LogOutput = $"CRITICAL EMERGENCY RECOVERY DATA — OAuth2 rotation partial failure. " +
|
||||
$"Old clientId={currentClientId} (DELETED). " +
|
||||
$"New clientId={newClientId}. New secret={newSecret}. " +
|
||||
$"These credentials must be stored manually. Error: {ex.Message}",
|
||||
};
|
||||
db.JobSteps.Add(emergencyStep);
|
||||
await db.SaveChangesAsync(CancellationToken.None);
|
||||
|
||||
throw; // Re-throw to fail the job
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"RotateCredentialsPipeline completed for job {JobId} (abbrev={Abbrev}, newClientId={ClientId})",
|
||||
job.Id, abbrev, newClientId);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static void EnsureSuccess<T>(Refit.IApiResponse<T> response)
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException(
|
||||
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
|
||||
}
|
||||
}
|
||||
96
OTSSignsOrchestrator.Server/Workers/StepRunner.cs
Normal file
96
OTSSignsOrchestrator.Server/Workers/StepRunner.cs
Normal file
@@ -0,0 +1,96 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Helper that wraps pipeline step execution with <see cref="JobStep"/> lifecycle management:
|
||||
/// creates the row, sets Running, captures output, marks Completed/Failed, and broadcasts
|
||||
/// progress via SignalR.
|
||||
/// </summary>
|
||||
public sealed class StepRunner
|
||||
{
|
||||
private readonly OrchestratorDbContext _db;
|
||||
private readonly IHubContext<FleetHub, IFleetClient> _hub;
|
||||
private readonly ILogger _logger;
|
||||
private readonly Guid _jobId;
|
||||
private readonly int _totalSteps;
|
||||
private int _currentStep;
|
||||
|
||||
public StepRunner(
|
||||
OrchestratorDbContext db,
|
||||
IHubContext<FleetHub, IFleetClient> hub,
|
||||
ILogger logger,
|
||||
Guid jobId,
|
||||
int totalSteps)
|
||||
{
|
||||
_db = db;
|
||||
_hub = hub;
|
||||
_logger = logger;
|
||||
_jobId = jobId;
|
||||
_totalSteps = totalSteps;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Execute a named step, persisting a <see cref="JobStep"/> record and broadcasting progress.
|
||||
/// </summary>
|
||||
/// <param name="stepName">Short identifier for the step (e.g. "mysql-setup").</param>
|
||||
/// <param name="action">
|
||||
/// Async delegate that performs the work. Return a log string summarising what happened.
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
public async Task RunAsync(string stepName, Func<Task<string>> action, CancellationToken ct)
|
||||
{
|
||||
_currentStep++;
|
||||
var pct = (int)((double)_currentStep / _totalSteps * 100);
|
||||
|
||||
var step = new JobStep
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
JobId = _jobId,
|
||||
StepName = stepName,
|
||||
Status = JobStepStatus.Running,
|
||||
StartedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
_db.JobSteps.Add(step);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
_logger.LogInformation("Job {JobId}: step [{Step}/{Total}] {StepName} started",
|
||||
_jobId, _currentStep, _totalSteps, stepName);
|
||||
|
||||
await _hub.Clients.All.SendJobProgressUpdate(
|
||||
_jobId.ToString(), stepName, pct, $"Starting {stepName}…");
|
||||
|
||||
try
|
||||
{
|
||||
var logOutput = await action();
|
||||
|
||||
step.Status = JobStepStatus.Completed;
|
||||
step.LogOutput = logOutput;
|
||||
step.CompletedAt = DateTime.UtcNow;
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
_logger.LogInformation("Job {JobId}: step {StepName} completed", _jobId, stepName);
|
||||
|
||||
await _hub.Clients.All.SendJobProgressUpdate(
|
||||
_jobId.ToString(), stepName, pct, logOutput);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
step.Status = JobStepStatus.Failed;
|
||||
step.LogOutput = ex.Message;
|
||||
step.CompletedAt = DateTime.UtcNow;
|
||||
await _db.SaveChangesAsync(CancellationToken.None);
|
||||
|
||||
_logger.LogError(ex, "Job {JobId}: step {StepName} failed", _jobId, stepName);
|
||||
|
||||
await _hub.Clients.All.SendJobProgressUpdate(
|
||||
_jobId.ToString(), stepName, pct, $"FAILED: {ex.Message}");
|
||||
|
||||
throw; // re-throw to fail the job
|
||||
}
|
||||
}
|
||||
}
|
||||
195
OTSSignsOrchestrator.Server/Workers/SuspendPipeline.cs
Normal file
195
OTSSignsOrchestrator.Server/Workers/SuspendPipeline.cs
Normal file
@@ -0,0 +1,195 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Renci.SshNet;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Subscription suspension pipeline — scales down Docker services and marks status.
|
||||
/// Handles <c>JobType = "suspend"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. scale-down — SSH docker service scale web=0, xmr=0
|
||||
/// 2. update-status — Customer.Status = Suspended, Instance.HealthStatus = Degraded
|
||||
/// 3. audit-log — Append-only AuditLog entry
|
||||
/// 4. broadcast — InstanceStatusChanged via FleetHub
|
||||
/// </summary>
|
||||
public sealed class SuspendPipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "suspend";
|
||||
|
||||
private const int TotalSteps = 4;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<SuspendPipeline> _logger;
|
||||
|
||||
public SuspendPipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<SuspendPipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
|
||||
// ── Step 1: scale-down ──────────────────────────────────────────────
|
||||
await runner.RunAsync("scale-down", async () =>
|
||||
{
|
||||
var sshHost = await GetSwarmSshHostAsync(settings);
|
||||
using var sshClient = CreateSshClient(sshHost);
|
||||
sshClient.Connect();
|
||||
|
||||
try
|
||||
{
|
||||
var cmd = $"docker service scale xibo-{abbrev}_web=0 xibo-{abbrev}_xmr=0";
|
||||
var result = RunSshCommand(sshClient, cmd);
|
||||
return $"Scaled down services for xibo-{abbrev}. Output: {result}";
|
||||
}
|
||||
finally
|
||||
{
|
||||
sshClient.Disconnect();
|
||||
}
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: update-status ───────────────────────────────────────────
|
||||
await runner.RunAsync("update-status", async () =>
|
||||
{
|
||||
var customer = await db.Customers.FirstOrDefaultAsync(c => c.Id == ctx.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {ctx.CustomerId} not found.");
|
||||
var instance = await db.Instances.FirstOrDefaultAsync(i => i.Id == ctx.InstanceId, ct)
|
||||
?? throw new InvalidOperationException($"Instance {ctx.InstanceId} not found.");
|
||||
|
||||
customer.Status = CustomerStatus.Suspended;
|
||||
instance.HealthStatus = HealthStatus.Degraded;
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return $"Customer '{abbrev}' status → Suspended, instance health → Degraded.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: audit-log ───────────────────────────────────────────────
|
||||
await runner.RunAsync("audit-log", async () =>
|
||||
{
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/stripe-webhook",
|
||||
Action = "suspend",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"Subscription suspended. Services scaled to 0. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return "AuditLog entry written for suspension.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 4: broadcast ───────────────────────────────────────────────
|
||||
await runner.RunAsync("broadcast", async () =>
|
||||
{
|
||||
await hub.Clients.All.SendInstanceStatusChanged(
|
||||
ctx.CustomerId.ToString(), CustomerStatus.Suspended.ToString());
|
||||
return "Broadcast InstanceStatusChanged → Suspended.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation("SuspendPipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers (shared pattern from Phase1Pipeline)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<SshConnectionInfo> GetSwarmSshHostAsync(SettingsService settings)
|
||||
{
|
||||
var host = await settings.GetAsync("Ssh.SwarmHost")
|
||||
?? throw new InvalidOperationException("SSH Swarm host not configured (Ssh.SwarmHost).");
|
||||
var portStr = await settings.GetAsync("Ssh.SwarmPort", "22");
|
||||
var user = await settings.GetAsync("Ssh.SwarmUser", "root");
|
||||
var keyPath = await settings.GetAsync("Ssh.SwarmKeyPath");
|
||||
var password = await settings.GetAsync("Ssh.SwarmPassword");
|
||||
|
||||
if (!int.TryParse(portStr, out var port)) port = 22;
|
||||
|
||||
return new SshConnectionInfo(host, port, user, keyPath, password);
|
||||
}
|
||||
|
||||
private static SshClient CreateSshClient(SshConnectionInfo info)
|
||||
{
|
||||
var authMethods = new List<AuthenticationMethod>();
|
||||
|
||||
if (!string.IsNullOrEmpty(info.KeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(info.KeyPath)));
|
||||
|
||||
if (!string.IsNullOrEmpty(info.Password))
|
||||
authMethods.Add(new PasswordAuthenticationMethod(info.Username, info.Password));
|
||||
|
||||
if (authMethods.Count == 0)
|
||||
{
|
||||
var defaultKeyPath = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".ssh", "id_rsa");
|
||||
if (File.Exists(defaultKeyPath))
|
||||
authMethods.Add(new PrivateKeyAuthenticationMethod(info.Username, new PrivateKeyFile(defaultKeyPath)));
|
||||
else
|
||||
throw new InvalidOperationException(
|
||||
$"No SSH authentication method available for {info.Host}:{info.Port}.");
|
||||
}
|
||||
|
||||
var connInfo = new Renci.SshNet.ConnectionInfo(info.Host, info.Port, info.Username, authMethods.ToArray());
|
||||
return new SshClient(connInfo);
|
||||
}
|
||||
|
||||
private static string RunSshCommand(SshClient client, string command)
|
||||
{
|
||||
using var cmd = client.RunCommand(command);
|
||||
if (cmd.ExitStatus != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"SSH command failed (exit {cmd.ExitStatus}): {cmd.Error}");
|
||||
return cmd.Result;
|
||||
}
|
||||
|
||||
internal sealed record SshConnectionInfo(
|
||||
string Host, int Port, string Username, string? KeyPath, string? Password);
|
||||
}
|
||||
189
OTSSignsOrchestrator.Server/Workers/UpdateScreenLimitPipeline.cs
Normal file
189
OTSSignsOrchestrator.Server/Workers/UpdateScreenLimitPipeline.cs
Normal file
@@ -0,0 +1,189 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using OTSSignsOrchestrator.Server.Clients;
|
||||
using OTSSignsOrchestrator.Server.Data;
|
||||
using OTSSignsOrchestrator.Server.Data.Entities;
|
||||
using OTSSignsOrchestrator.Server.Hubs;
|
||||
using OTSSignsOrchestrator.Core.Services;
|
||||
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Updates the Xibo CMS screen limit and records a snapshot.
|
||||
/// Handles <c>JobType = "update-screen-limit"</c>.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. update-settings — PUT /api/settings with new MAX_LICENSED_DISPLAYS
|
||||
/// 2. update-snapshot — Record new screen count in ScreenSnapshots for today
|
||||
/// 3. audit-log — Append-only AuditLog entry
|
||||
/// </summary>
|
||||
public sealed class UpdateScreenLimitPipeline : IProvisioningPipeline
|
||||
{
|
||||
public string HandlesJobType => "update-screen-limit";
|
||||
|
||||
private const int TotalSteps = 3;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<UpdateScreenLimitPipeline> _logger;
|
||||
|
||||
public UpdateScreenLimitPipeline(
|
||||
IServiceProvider services,
|
||||
ILogger<UpdateScreenLimitPipeline> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||||
{
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||||
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||||
var xiboFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||||
|
||||
var ctx = await BuildContextAsync(job, db, ct);
|
||||
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||||
|
||||
var abbrev = ctx.Abbreviation;
|
||||
|
||||
// Parse newScreenCount from Job.Parameters JSON
|
||||
var newScreenCount = ParseScreenCount(ctx.ParametersJson)
|
||||
?? throw new InvalidOperationException(
|
||||
"Job.Parameters must contain 'newScreenCount' (integer) for update-screen-limit.");
|
||||
|
||||
// Get Xibo API client via XiboClientFactory
|
||||
var oauthReg = await db.OauthAppRegistries
|
||||
.Where(r => r.InstanceId == ctx.InstanceId)
|
||||
.OrderByDescending(r => r.CreatedAt)
|
||||
.FirstOrDefaultAsync(ct)
|
||||
?? throw new InvalidOperationException(
|
||||
$"No OauthAppRegistry found for instance {ctx.InstanceId}.");
|
||||
|
||||
var oauthSecret = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev))
|
||||
?? throw new InvalidOperationException(
|
||||
$"OAuth secret not found in settings for instance '{abbrev}'.");
|
||||
|
||||
var xiboClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, oauthReg.ClientId, oauthSecret);
|
||||
|
||||
// ── Step 1: update-settings ─────────────────────────────────────────
|
||||
await runner.RunAsync("update-settings", async () =>
|
||||
{
|
||||
var response = await xiboClient.UpdateSettingsAsync(new UpdateSettingsRequest(
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["MAX_LICENSED_DISPLAYS"] = newScreenCount.ToString(),
|
||||
}));
|
||||
|
||||
EnsureSuccess(response);
|
||||
return $"Xibo MAX_LICENSED_DISPLAYS updated to {newScreenCount} for {ctx.InstanceUrl}.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 2: update-snapshot ─────────────────────────────────────────
|
||||
await runner.RunAsync("update-snapshot", async () =>
|
||||
{
|
||||
var today = DateOnly.FromDateTime(DateTime.UtcNow);
|
||||
|
||||
// Upsert: if a snapshot already exists for today, update it; otherwise insert
|
||||
var existing = await db.ScreenSnapshots
|
||||
.FirstOrDefaultAsync(s => s.InstanceId == ctx.InstanceId && s.SnapshotDate == today, ct);
|
||||
|
||||
if (existing is not null)
|
||||
{
|
||||
existing.ScreenCount = newScreenCount;
|
||||
}
|
||||
else
|
||||
{
|
||||
db.ScreenSnapshots.Add(new ScreenSnapshot
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
SnapshotDate = today,
|
||||
ScreenCount = newScreenCount,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
}
|
||||
|
||||
// Also update Customer.ScreenCount to reflect the new limit
|
||||
var customer = await db.Customers.FirstOrDefaultAsync(c => c.Id == ctx.CustomerId, ct);
|
||||
if (customer is not null)
|
||||
customer.ScreenCount = newScreenCount;
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
return $"ScreenSnapshot recorded for {today}: {newScreenCount} screens.";
|
||||
}, ct);
|
||||
|
||||
// ── Step 3: audit-log ───────────────────────────────────────────────
|
||||
await runner.RunAsync("audit-log", async () =>
|
||||
{
|
||||
db.AuditLogs.Add(new AuditLog
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
InstanceId = ctx.InstanceId,
|
||||
Actor = "system/stripe-webhook",
|
||||
Action = "update-screen-limit",
|
||||
Target = $"xibo-{abbrev}",
|
||||
Outcome = "success",
|
||||
Detail = $"Screen limit updated to {newScreenCount}. Job {job.Id}.",
|
||||
OccurredAt = DateTime.UtcNow,
|
||||
});
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
return "AuditLog entry written for screen limit update.";
|
||||
}, ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"UpdateScreenLimitPipeline completed for job {JobId} (abbrev={Abbrev}, screens={Count})",
|
||||
job.Id, abbrev, newScreenCount);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private static int? ParseScreenCount(string? parametersJson)
|
||||
{
|
||||
if (string.IsNullOrEmpty(parametersJson)) return null;
|
||||
|
||||
using var doc = JsonDocument.Parse(parametersJson);
|
||||
if (doc.RootElement.TryGetProperty("newScreenCount", out var prop) && prop.TryGetInt32(out var count))
|
||||
return count;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||||
{
|
||||
var customer = await db.Customers
|
||||
.Include(c => c.Instances)
|
||||
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||||
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||||
|
||||
var instance = customer.Instances.FirstOrDefault()
|
||||
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||||
|
||||
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||||
|
||||
return new PipelineContext
|
||||
{
|
||||
JobId = job.Id,
|
||||
CustomerId = customer.Id,
|
||||
InstanceId = instance.Id,
|
||||
Abbreviation = abbrev,
|
||||
CompanyName = customer.CompanyName,
|
||||
AdminEmail = customer.AdminEmail,
|
||||
AdminFirstName = customer.AdminFirstName,
|
||||
InstanceUrl = instance.XiboUrl,
|
||||
DockerStackName = instance.DockerStackName,
|
||||
ParametersJson = job.Parameters,
|
||||
};
|
||||
}
|
||||
|
||||
private static void EnsureSuccess<T>(Refit.IApiResponse<T> response)
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException(
|
||||
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
|
||||
}
|
||||
}
|
||||
139
OTSSignsOrchestrator.Server/Workers/XiboFeatureManifests.cs
Normal file
139
OTSSignsOrchestrator.Server/Workers/XiboFeatureManifests.cs
Normal file
@@ -0,0 +1,139 @@
|
||||
namespace OTSSignsOrchestrator.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Hardcoded Xibo feature ACL manifests per role.
|
||||
/// Used by Phase2Pipeline step "assign-group-acl" when calling
|
||||
/// <c>POST /api/group/{id}/acl</c>.
|
||||
///
|
||||
/// ObjectId is the feature key, PermissionsId is the permission level ("view", "edit", "delete").
|
||||
/// </summary>
|
||||
public static class XiboFeatureManifests
|
||||
{
|
||||
/// <summary>Viewer role: read-only access to layouts, displays, media.</summary>
|
||||
public static readonly string[] ViewerObjectIds =
|
||||
[
|
||||
"layout.view",
|
||||
"media.view",
|
||||
"display.view",
|
||||
"schedule.view",
|
||||
"report.view",
|
||||
];
|
||||
|
||||
public static readonly string[] ViewerPermissionIds =
|
||||
[
|
||||
"view",
|
||||
"view",
|
||||
"view",
|
||||
"view",
|
||||
"view",
|
||||
];
|
||||
|
||||
/// <summary>Editor role: view + edit for layouts, media, schedules.</summary>
|
||||
public static readonly string[] EditorObjectIds =
|
||||
[
|
||||
"layout.view",
|
||||
"layout.edit",
|
||||
"media.view",
|
||||
"media.edit",
|
||||
"display.view",
|
||||
"schedule.view",
|
||||
"schedule.edit",
|
||||
"report.view",
|
||||
];
|
||||
|
||||
public static readonly string[] EditorPermissionIds =
|
||||
[
|
||||
"view",
|
||||
"edit",
|
||||
"view",
|
||||
"edit",
|
||||
"view",
|
||||
"view",
|
||||
"edit",
|
||||
"view",
|
||||
];
|
||||
|
||||
/// <summary>Admin role: full access to all features.</summary>
|
||||
public static readonly string[] AdminObjectIds =
|
||||
[
|
||||
"layout.view",
|
||||
"layout.edit",
|
||||
"layout.delete",
|
||||
"media.view",
|
||||
"media.edit",
|
||||
"media.delete",
|
||||
"display.view",
|
||||
"display.edit",
|
||||
"display.delete",
|
||||
"schedule.view",
|
||||
"schedule.edit",
|
||||
"schedule.delete",
|
||||
"report.view",
|
||||
"user.view",
|
||||
"user.edit",
|
||||
];
|
||||
|
||||
public static readonly string[] AdminPermissionIds =
|
||||
[
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"view",
|
||||
"edit",
|
||||
];
|
||||
|
||||
/// <summary>OTS IT group: full super-admin access (all features + user management).</summary>
|
||||
public static readonly string[] OtsItObjectIds =
|
||||
[
|
||||
"layout.view",
|
||||
"layout.edit",
|
||||
"layout.delete",
|
||||
"media.view",
|
||||
"media.edit",
|
||||
"media.delete",
|
||||
"display.view",
|
||||
"display.edit",
|
||||
"display.delete",
|
||||
"schedule.view",
|
||||
"schedule.edit",
|
||||
"schedule.delete",
|
||||
"report.view",
|
||||
"user.view",
|
||||
"user.edit",
|
||||
"user.delete",
|
||||
"application.view",
|
||||
"application.edit",
|
||||
];
|
||||
|
||||
public static readonly string[] OtsItPermissionIds =
|
||||
[
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"view",
|
||||
"edit",
|
||||
"delete",
|
||||
"view",
|
||||
"edit",
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user