feat: Implement provisioning pipelines for subscription management

- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes.
- Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging.
- Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR.
- Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes.
- Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots.
- Introduce XiboFeatureManifests for hardcoded feature ACLs per role.
- Add docker-compose.dev.yml for local development with PostgreSQL setup.
This commit is contained in:
Matt Batchelder
2026-03-18 10:27:26 -04:00
parent c2e03de8bb
commit c6d46098dd
77 changed files with 9412 additions and 29 deletions

View File

@@ -0,0 +1,479 @@
using System.Security.Cryptography;
using System.Text.Json;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Quartz;
using OTSSignsOrchestrator.Core.Services;
using OTSSignsOrchestrator.Server.Clients;
using OTSSignsOrchestrator.Server.Data;
using OTSSignsOrchestrator.Server.Data.Entities;
using OTSSignsOrchestrator.Server.Hubs;
using OTSSignsOrchestrator.Server.Services;
namespace OTSSignsOrchestrator.Server.Workers;
/// <summary>
/// Phase 2 provisioning pipeline — Xibo CMS bootstrap. Handles <c>JobType = "bootstrap"</c>.
/// Triggered after the stack deployed in Phase 1 becomes healthy.
///
/// Steps:
/// 1. xibo-health-check — Poll GET /about until 200 OK
/// 2. create-ots-admin — POST /api/user (superAdmin)
/// 3. create-ots-svc — POST /api/user (service account)
/// 4. create-oauth2-app — POST /api/application — IMMEDIATELY capture secret
/// 5. create-groups — POST /api/group (viewer, editor, admin, ots-it)
/// 6. assign-group-acl — POST /api/group/{id}/acl per role
/// 7. assign-service-accounts — Assign admin + svc to ots-it group
/// 8. apply-theme — PUT /api/settings THEME_NAME=otssigns
/// 9. delete-default-user — Delete xibo_admin (after safety check)
/// 10. schedule-snapshot — Register Quartz DailySnapshotJob
/// 11. authentik-invite — Create Authentik user + invitation, send welcome email
/// </summary>
public sealed class Phase2Pipeline : IProvisioningPipeline
{
public string HandlesJobType => "bootstrap";
private const int TotalSteps = 11;
private readonly IServiceProvider _services;
private readonly ILogger<Phase2Pipeline> _logger;
public Phase2Pipeline(
IServiceProvider services,
ILogger<Phase2Pipeline> logger)
{
_services = services;
_logger = logger;
}
public async Task ExecuteAsync(Job job, CancellationToken ct)
{
await using var scope = _services.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
var xiboFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
var authentikClient = scope.ServiceProvider.GetRequiredService<IAuthentikClient>();
var bws = scope.ServiceProvider.GetRequiredService<IBitwardenSecretService>();
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
var emailService = scope.ServiceProvider.GetRequiredService<EmailService>();
var schedulerFactory = scope.ServiceProvider.GetRequiredService<ISchedulerFactory>();
var ctx = await BuildContextAsync(job, db, ct);
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
var abbrev = ctx.Abbreviation;
var instanceUrl = ctx.InstanceUrl;
// The initial Xibo CMS ships with xibo_admin / password credentials.
// We use these to bootstrap via the API.
IXiboApiClient? xiboClient = null;
// Mutable state accumulated across steps
var adminPassword = GenerateRandomPassword(24);
var svcPassword = GenerateRandomPassword(24);
int otsAdminUserId = 0;
int otsSvcUserId = 0;
string otsOAuthClientId = string.Empty;
string otsOAuthClientSecret = string.Empty;
var groupIds = new Dictionary<string, int>(); // groupName → groupId
// ── Step 1: xibo-health-check ───────────────────────────────────────
await runner.RunAsync("xibo-health-check", async () =>
{
// Poll GET /about every 10 seconds, up to 5 minutes
var timeout = TimeSpan.FromMinutes(5);
var interval = TimeSpan.FromSeconds(10);
var deadline = DateTimeOffset.UtcNow.Add(timeout);
// Create a temporary client using default bootstrap credentials
// xibo_admin/password → OAuth client_credentials from the seed application
// The first call is to /about which doesn't need auth, use a raw HttpClient
using var httpClient = new HttpClient { BaseAddress = new Uri(instanceUrl.TrimEnd('/')) };
while (DateTimeOffset.UtcNow < deadline)
{
ct.ThrowIfCancellationRequested();
try
{
var response = await httpClient.GetAsync("/api/about", ct);
if (response.IsSuccessStatusCode)
{
return $"Xibo CMS at {instanceUrl} is healthy (status {(int)response.StatusCode}).";
}
}
catch (HttpRequestException)
{
// Not ready yet
}
await Task.Delay(interval, ct);
}
throw new TimeoutException(
$"Xibo CMS at {instanceUrl} did not return 200 OK from /about within {timeout.TotalMinutes} minutes.");
}, ct);
// Get a Refit client using the seed OAuth app credentials (xibo_admin bootstrap)
// Parameters JSON should contain bootstrapClientId + bootstrapClientSecret
var parameters = !string.IsNullOrEmpty(ctx.ParametersJson)
? JsonDocument.Parse(ctx.ParametersJson)
: null;
var bootstrapClientId = parameters?.RootElement.TryGetProperty("bootstrapClientId", out var bcid) == true
? bcid.GetString() ?? string.Empty
: string.Empty;
var bootstrapClientSecret = parameters?.RootElement.TryGetProperty("bootstrapClientSecret", out var bcs) == true
? bcs.GetString() ?? string.Empty
: string.Empty;
if (string.IsNullOrEmpty(bootstrapClientId) || string.IsNullOrEmpty(bootstrapClientSecret))
throw new InvalidOperationException(
"Bootstrap OAuth credentials (bootstrapClientId, bootstrapClientSecret) must be provided in Job.Parameters.");
xiboClient = await xiboFactory.CreateAsync(instanceUrl, bootstrapClientId, bootstrapClientSecret);
// ── Step 2: create-ots-admin ────────────────────────────────────────
await runner.RunAsync("create-ots-admin", async () =>
{
var username = $"ots-admin-{abbrev}";
var email = $"ots-admin-{abbrev}@ots-signs.com";
var response = await xiboClient.CreateUserAsync(new CreateUserRequest(
UserName: username,
Email: email,
Password: adminPassword,
UserTypeId: 1, // SuperAdmin
HomePageId: 1));
EnsureSuccess(response);
var data = response.Content
?? throw new InvalidOperationException("CreateUser returned empty response.");
otsAdminUserId = Convert.ToInt32(data["userId"]);
return $"Created user '{username}' (userId={otsAdminUserId}, type=SuperAdmin).";
}, ct);
// ── Step 3: create-ots-svc ──────────────────────────────────────────
await runner.RunAsync("create-ots-svc", async () =>
{
var username = $"ots-svc-{abbrev}";
var email = $"ots-svc-{abbrev}@ots-signs.com";
var response = await xiboClient.CreateUserAsync(new CreateUserRequest(
UserName: username,
Email: email,
Password: svcPassword,
UserTypeId: 1, // SuperAdmin — service account needs full API access
HomePageId: 1));
EnsureSuccess(response);
var data = response.Content
?? throw new InvalidOperationException("CreateUser returned empty response.");
otsSvcUserId = Convert.ToInt32(data["userId"]);
return $"Created user '{username}' (userId={otsSvcUserId}, type=SuperAdmin).";
}, ct);
// ── Step 4: create-oauth2-app ───────────────────────────────────────
// CRITICAL: POST /api/application — NOT GET (that's blocked).
// The OAuth2 client secret is returned ONLY in this response. Capture immediately.
await runner.RunAsync("create-oauth2-app", async () =>
{
var appName = $"OTS Orchestrator — {abbrev.ToUpperInvariant()}";
var response = await xiboClient.CreateApplicationAsync(new CreateApplicationRequest(appName));
EnsureSuccess(response);
var data = response.Content
?? throw new InvalidOperationException("CreateApplication returned empty response.");
// CRITICAL: Capture secret immediately — it cannot be retrieved again.
otsOAuthClientId = data["key"]?.ToString()
?? throw new InvalidOperationException("OAuth application response missing 'key'.");
otsOAuthClientSecret = data["secret"]?.ToString()
?? throw new InvalidOperationException("OAuth application response missing 'secret'.");
// Store to Bitwarden CLI wrapper
await bws.CreateInstanceSecretAsync(
key: $"{abbrev}/xibo-oauth-secret",
value: otsOAuthClientSecret,
note: $"Xibo CMS OAuth2 client secret for OTS application. ClientId: {otsOAuthClientId}");
// Store clientId ONLY in the database — NEVER store the secret in the DB
db.OauthAppRegistries.Add(new OauthAppRegistry
{
Id = Guid.NewGuid(),
InstanceId = ctx.InstanceId,
ClientId = otsOAuthClientId,
CreatedAt = DateTime.UtcNow,
});
await db.SaveChangesAsync(ct);
return $"OAuth2 app '{appName}' created. ClientId={otsOAuthClientId}. Secret stored in Bitwarden (NEVER in DB).";
}, ct);
// ── Step 5: create-groups ───────────────────────────────────────────
await runner.RunAsync("create-groups", async () =>
{
var groupDefs = new[]
{
($"{abbrev}-viewer", "Viewer role"),
($"{abbrev}-editor", "Editor role"),
($"{abbrev}-admin", "Admin role"),
($"ots-it-{abbrev}", "OTS IT internal"),
};
foreach (var (name, description) in groupDefs)
{
var response = await xiboClient.CreateGroupAsync(new CreateGroupRequest(name, description));
EnsureSuccess(response);
var data = response.Content
?? throw new InvalidOperationException($"CreateGroup '{name}' returned empty response.");
groupIds[name] = Convert.ToInt32(data["groupId"]);
}
return $"Created {groupDefs.Length} groups: {string.Join(", ", groupIds.Keys)}.";
}, ct);
// ── Step 6: assign-group-acl ────────────────────────────────────────
// POST /api/group/{id}/acl — NOT /features
await runner.RunAsync("assign-group-acl", async () =>
{
var aclMap = new Dictionary<string, (string[] ObjectIds, string[] PermissionIds)>
{
[$"{abbrev}-viewer"] = (XiboFeatureManifests.ViewerObjectIds, XiboFeatureManifests.ViewerPermissionIds),
[$"{abbrev}-editor"] = (XiboFeatureManifests.EditorObjectIds, XiboFeatureManifests.EditorPermissionIds),
[$"{abbrev}-admin"] = (XiboFeatureManifests.AdminObjectIds, XiboFeatureManifests.AdminPermissionIds),
[$"ots-it-{abbrev}"] = (XiboFeatureManifests.OtsItObjectIds, XiboFeatureManifests.OtsItPermissionIds),
};
var applied = new List<string>();
foreach (var (groupName, (objectIds, permissionIds)) in aclMap)
{
if (!groupIds.TryGetValue(groupName, out var groupId))
throw new InvalidOperationException($"Group '{groupName}' ID not found.");
var response = await xiboClient.SetGroupAclAsync(groupId, new SetAclRequest(objectIds, permissionIds));
EnsureSuccess(response);
applied.Add($"{groupName} ({objectIds.Length} features)");
}
return $"ACL assigned: {string.Join(", ", applied)}.";
}, ct);
// ── Step 7: assign-service-accounts ─────────────────────────────────
await runner.RunAsync("assign-service-accounts", async () =>
{
var otsItGroupName = $"ots-it-{abbrev}";
if (!groupIds.TryGetValue(otsItGroupName, out var otsItGroupId))
throw new InvalidOperationException($"Group '{otsItGroupName}' ID not found.");
// Assign both ots-admin and ots-svc to the ots-it group
var response = await xiboClient.AssignUserToGroupAsync(
otsItGroupId,
new AssignMemberRequest([otsAdminUserId, otsSvcUserId]));
EnsureSuccess(response);
return $"Assigned ots-admin-{abbrev} (id={otsAdminUserId}) and ots-svc-{abbrev} (id={otsSvcUserId}) to group '{otsItGroupName}'.";
}, ct);
// ── Step 8: apply-theme ─────────────────────────────────────────────
await runner.RunAsync("apply-theme", async () =>
{
var response = await xiboClient.UpdateSettingsAsync(new UpdateSettingsRequest(
new Dictionary<string, string> { ["THEME_NAME"] = "otssigns" }));
EnsureSuccess(response);
return "Theme set to 'otssigns'.";
}, ct);
// ── Step 9: delete-default-user ─────────────────────────────────────
await runner.RunAsync("delete-default-user", async () =>
{
// First verify ots-admin works via OAuth2 client_credentials test
var testClient = await xiboFactory.CreateAsync(instanceUrl, otsOAuthClientId, otsOAuthClientSecret);
var aboutResponse = await testClient.GetAboutAsync();
if (!aboutResponse.IsSuccessStatusCode)
throw new InvalidOperationException(
$"OAuth2 verification failed for ots-admin app (status {aboutResponse.StatusCode}). " +
"Refusing to delete xibo_admin — it may be the only working admin account.");
// Use GetAllPagesAsync to find xibo_admin user — default page size is only 10
var allUsers = await xiboClient.GetAllPagesAsync(
(start, length) => xiboClient.GetUsersAsync(start, length));
var xiboAdminUser = allUsers.FirstOrDefault(u =>
u.TryGetValue("userName", out var name) &&
string.Equals(name?.ToString(), "xibo_admin", StringComparison.OrdinalIgnoreCase));
if (xiboAdminUser is null)
return "xibo_admin user not found — may have already been deleted.";
var xiboAdminId = Convert.ToInt32(xiboAdminUser["userId"]);
// Safety check: never delete if it would be the last SuperAdmin
var superAdminCount = allUsers.Count(u =>
u.TryGetValue("userTypeId", out var typeId) &&
Convert.ToInt32(typeId) == 1);
if (superAdminCount <= 1)
throw new InvalidOperationException(
"Cannot delete xibo_admin — it is the only SuperAdmin. " +
"Ensure ots-admin was created as SuperAdmin before retrying.");
await xiboClient.DeleteUserAsync(xiboAdminId);
return $"Deleted xibo_admin (userId={xiboAdminId}). Remaining SuperAdmins: {superAdminCount - 1}.";
}, ct);
// ── Step 10: schedule-snapshot ──────────────────────────────────────
await runner.RunAsync("schedule-snapshot", async () =>
{
var scheduler = await schedulerFactory.GetScheduler(ct);
var jobKey = new JobKey($"snapshot-{abbrev}", "daily-snapshots");
var triggerKey = new TriggerKey($"snapshot-trigger-{abbrev}", "daily-snapshots");
var quartzJob = JobBuilder.Create<DailySnapshotJob>()
.WithIdentity(jobKey)
.UsingJobData("abbrev", abbrev)
.UsingJobData("instanceId", ctx.InstanceId.ToString())
.StoreDurably()
.Build();
var trigger = TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithCronSchedule("0 0 2 * * ?") // 2:00 AM daily
.Build();
await scheduler.ScheduleJob(quartzJob, trigger, ct);
return $"DailySnapshotJob scheduled for instance '{abbrev}' — daily at 02:00 UTC.";
}, ct);
// ── Step 11: authentik-invite ───────────────────────────────────────
await runner.RunAsync("authentik-invite", async () =>
{
var adminEmail = ctx.AdminEmail;
var firstName = ctx.AdminFirstName;
// Create Authentik user
var userResponse = await authentikClient.CreateUserAsync(new CreateAuthentikUserRequest(
Username: adminEmail,
Name: $"{firstName}",
Email: adminEmail,
Groups: [$"customer-{abbrev}", $"customer-{abbrev}-viewer"]));
EnsureSuccess(userResponse);
// Create invitation with 7-day expiry
var inviteResponse = await authentikClient.CreateInvitationAsync(new CreateFlowRequest(
Name: $"invite-{abbrev}",
SingleUse: true,
Expires: DateTimeOffset.UtcNow.AddDays(7)));
EnsureSuccess(inviteResponse);
var inviteData = inviteResponse.Content;
var invitationLink = inviteData?.TryGetValue("pk", out var pk) == true
? $"{instanceUrl}/if/flow/invite-{abbrev}/?itoken={pk}"
: "(invitation link unavailable)";
// Send welcome email
await emailService.SendWelcomeEmailAsync(adminEmail, firstName, instanceUrl, invitationLink);
return $"Authentik user '{adminEmail}' created, assigned to customer-{abbrev} + customer-{abbrev}-viewer. " +
$"Invitation link: {invitationLink}. Welcome email sent.";
}, ct);
_logger.LogInformation("Phase2Pipeline completed for job {JobId} (abbrev={Abbrev})", job.Id, abbrev);
}
// ─────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
{
var customer = await db.Customers
.Include(c => c.Instances)
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
var instance = customer.Instances.FirstOrDefault()
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
var abbrev = customer.Abbreviation.ToLowerInvariant();
return new PipelineContext
{
JobId = job.Id,
CustomerId = customer.Id,
InstanceId = instance.Id,
Abbreviation = abbrev,
CompanyName = customer.CompanyName,
AdminEmail = customer.AdminEmail,
AdminFirstName = customer.AdminFirstName,
InstanceUrl = instance.XiboUrl,
DockerStackName = instance.DockerStackName,
ParametersJson = job.Parameters,
};
}
private static string GenerateRandomPassword(int length)
{
const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*";
return RandomNumberGenerator.GetString(chars, length);
}
private static void EnsureSuccess<T>(Refit.IApiResponse<T> response)
{
if (!response.IsSuccessStatusCode)
throw new InvalidOperationException(
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
}
}
/// <summary>
/// Quartz job placeholder for daily instance snapshots.
/// Registered per-instance by Phase2Pipeline step 10.
/// </summary>
[DisallowConcurrentExecution]
public sealed class DailySnapshotJob : IJob
{
private readonly IServiceProvider _services;
private readonly ILogger<DailySnapshotJob> _logger;
public DailySnapshotJob(IServiceProvider services, ILogger<DailySnapshotJob> logger)
{
_services = services;
_logger = logger;
}
public async Task Execute(IJobExecutionContext context)
{
var abbrev = context.MergedJobDataMap.GetString("abbrev");
var instanceIdStr = context.MergedJobDataMap.GetString("instanceId");
_logger.LogInformation("DailySnapshotJob running for instance {Abbrev} (id={InstanceId})",
abbrev, instanceIdStr);
await using var scope = _services.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
if (!Guid.TryParse(instanceIdStr, out var instanceId))
{
_logger.LogError("DailySnapshotJob: invalid instanceId '{InstanceId}'", instanceIdStr);
return;
}
db.ScreenSnapshots.Add(new ScreenSnapshot
{
Id = Guid.NewGuid(),
InstanceId = instanceId,
SnapshotDate = DateOnly.FromDateTime(DateTime.UtcNow),
ScreenCount = 0,
CreatedAt = DateTime.UtcNow,
});
await db.SaveChangesAsync();
_logger.LogInformation("DailySnapshotJob completed for instance {Abbrev}", abbrev);
}
}