275 lines
14 KiB
C#
275 lines
14 KiB
C#
|
using Microsoft.AspNetCore.SignalR;
|
||
|
|
using Microsoft.EntityFrameworkCore;
|
||
|
|
using OTSSignsOrchestrator.Core.Services;
|
||
|
|
using OTSSignsOrchestrator.Server.Clients;
|
||
|
|
using OTSSignsOrchestrator.Server.Data;
|
||
|
|
using OTSSignsOrchestrator.Server.Data.Entities;
|
||
|
|
using OTSSignsOrchestrator.Server.Hubs;
|
||
|
|
|
||
|
|
namespace OTSSignsOrchestrator.Server.Workers;
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// OAuth2 credential rotation pipeline — deletes the old Xibo OAuth app, creates a new one,
|
||
|
|
/// stores the new credentials, and verifies access. Handles <c>JobType = "rotate-oauth2"</c>.
|
||
|
|
///
|
||
|
|
/// CRITICAL: OAuth2 clientId CHANGES on rotation — there is no in-place secret refresh.
|
||
|
|
/// The secret is returned ONLY in the <c>POST /api/application</c> response.
|
||
|
|
///
|
||
|
|
/// Steps:
|
||
|
|
/// 1. delete-old-app — Delete current OAuth2 application via Xibo API
|
||
|
|
/// 2. create-new-app — POST /api/application — IMMEDIATELY capture secret
|
||
|
|
/// 3. store-credentials — Update Bitwarden + OauthAppRegistry with new clientId
|
||
|
|
/// 4. verify-access — Test new credentials via client_credentials flow
|
||
|
|
/// 5. audit-log — Write AuditLog (or CRITICAL error with emergency creds if steps 3-4 fail)
|
||
|
|
/// </summary>
|
||
|
|
public sealed class RotateCredentialsPipeline : IProvisioningPipeline
|
||
|
|
{
|
||
|
|
public string HandlesJobType => "rotate-oauth2";
|
||
|
|
|
||
|
|
private const int TotalSteps = 5;
|
||
|
|
|
||
|
|
private readonly IServiceProvider _services;
|
||
|
|
private readonly ILogger<RotateCredentialsPipeline> _logger;
|
||
|
|
|
||
|
|
public RotateCredentialsPipeline(
|
||
|
|
IServiceProvider services,
|
||
|
|
ILogger<RotateCredentialsPipeline> logger)
|
||
|
|
{
|
||
|
|
_services = services;
|
||
|
|
_logger = logger;
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task ExecuteAsync(Job job, CancellationToken ct)
|
||
|
|
{
|
||
|
|
await using var scope = _services.CreateAsyncScope();
|
||
|
|
var db = scope.ServiceProvider.GetRequiredService<OrchestratorDbContext>();
|
||
|
|
var hub = scope.ServiceProvider.GetRequiredService<IHubContext<FleetHub, IFleetClient>>();
|
||
|
|
var xiboFactory = scope.ServiceProvider.GetRequiredService<XiboClientFactory>();
|
||
|
|
var bws = scope.ServiceProvider.GetRequiredService<IBitwardenSecretService>();
|
||
|
|
var settings = scope.ServiceProvider.GetRequiredService<SettingsService>();
|
||
|
|
|
||
|
|
var ctx = await BuildContextAsync(job, db, ct);
|
||
|
|
var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps);
|
||
|
|
|
||
|
|
var abbrev = ctx.Abbreviation;
|
||
|
|
|
||
|
|
// Load current OAuth2 credentials
|
||
|
|
var currentReg = await db.OauthAppRegistries
|
||
|
|
.Where(r => r.InstanceId == ctx.InstanceId)
|
||
|
|
.OrderByDescending(r => r.CreatedAt)
|
||
|
|
.FirstOrDefaultAsync(ct)
|
||
|
|
?? throw new InvalidOperationException(
|
||
|
|
$"No OauthAppRegistry found for instance {ctx.InstanceId}. Cannot rotate.");
|
||
|
|
|
||
|
|
var currentSecret = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev))
|
||
|
|
?? throw new InvalidOperationException(
|
||
|
|
$"OAuth secret not found in settings for instance '{abbrev}'. Cannot rotate.");
|
||
|
|
|
||
|
|
var currentClientId = currentReg.ClientId;
|
||
|
|
|
||
|
|
// Get a Xibo API client using the current (about-to-be-deleted) credentials
|
||
|
|
var xiboClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, currentClientId, currentSecret);
|
||
|
|
|
||
|
|
// Mutable state — captured across steps
|
||
|
|
string newClientId = string.Empty;
|
||
|
|
string newSecret = string.Empty;
|
||
|
|
|
||
|
|
// ── Step 1: delete-old-app ──────────────────────────────────────────
|
||
|
|
await runner.RunAsync("delete-old-app", async () =>
|
||
|
|
{
|
||
|
|
await xiboClient.DeleteApplicationAsync(currentClientId);
|
||
|
|
return $"Old OAuth2 application deleted. ClientId={currentClientId}.";
|
||
|
|
}, ct);
|
||
|
|
|
||
|
|
// ── Step 2: create-new-app ──────────────────────────────────────────
|
||
|
|
// CRITICAL: After this step, the old credentials are gone. If subsequent steps fail,
|
||
|
|
// the new clientId + secret MUST be logged for manual recovery.
|
||
|
|
await runner.RunAsync("create-new-app", async () =>
|
||
|
|
{
|
||
|
|
// Need a client with some auth to create the new app. Use the instance's bootstrap
|
||
|
|
// admin credentials (ots-admin user) via direct login if available, or we can
|
||
|
|
// re-authenticate using the base URL. Since the old app is deleted, we need a
|
||
|
|
// different auth mechanism. The Xibo CMS allows creating apps as any authenticated
|
||
|
|
// super-admin. Use the ots-admin password from Bitwarden.
|
||
|
|
//
|
||
|
|
// However, the simplest path: the DELETE above invalidated the old token, but the
|
||
|
|
// Xibo CMS still has the ots-admin and ots-svc users. We stored the admin password
|
||
|
|
// in Bitwarden. Retrieve it and create a new factory client.
|
||
|
|
var adminPassBwsId = await settings.GetAsync(SettingsService.InstanceAdminPasswordSecretId(abbrev));
|
||
|
|
if (string.IsNullOrEmpty(adminPassBwsId))
|
||
|
|
throw new InvalidOperationException(
|
||
|
|
$"Admin password Bitwarden secret ID not found for '{abbrev}'. Cannot create new OAuth app.");
|
||
|
|
|
||
|
|
var adminPassSecret = await bws.GetSecretAsync(adminPassBwsId);
|
||
|
|
var bootstrapClientId = await settings.GetAsync(SettingsService.XiboBootstrapClientId)
|
||
|
|
?? throw new InvalidOperationException("Bootstrap OAuth client ID not configured.");
|
||
|
|
var bootstrapClientSecret = await settings.GetAsync(SettingsService.XiboBootstrapClientSecret)
|
||
|
|
?? throw new InvalidOperationException("Bootstrap OAuth client secret not configured.");
|
||
|
|
|
||
|
|
// Re-create client using bootstrap credentials
|
||
|
|
var bootstrapClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, bootstrapClientId, bootstrapClientSecret);
|
||
|
|
|
||
|
|
var appName = $"OTS Orchestrator — {abbrev.ToUpperInvariant()}";
|
||
|
|
var response = await bootstrapClient.CreateApplicationAsync(new CreateApplicationRequest(appName));
|
||
|
|
|
||
|
|
EnsureSuccess(response);
|
||
|
|
var data = response.Content
|
||
|
|
?? throw new InvalidOperationException("CreateApplication returned empty response.");
|
||
|
|
|
||
|
|
// CRITICAL: Capture secret IMMEDIATELY — it is returned ONLY in this response.
|
||
|
|
newClientId = data["key"]?.ToString()
|
||
|
|
?? throw new InvalidOperationException("OAuth application response missing 'key'.");
|
||
|
|
newSecret = data["secret"]?.ToString()
|
||
|
|
?? throw new InvalidOperationException("OAuth application response missing 'secret'.");
|
||
|
|
|
||
|
|
return $"New OAuth2 application created. ClientId={newClientId}. Secret captured (stored in next step).";
|
||
|
|
}, ct);
|
||
|
|
|
||
|
|
// Steps 3-5: if ANY of these fail after step 2, we must log the credentials for recovery.
|
||
|
|
// The old app is already deleted — there is no rollback path.
|
||
|
|
try
|
||
|
|
{
|
||
|
|
// ── Step 3: store-credentials ───────────────────────────────────
|
||
|
|
await runner.RunAsync("store-credentials", async () =>
|
||
|
|
{
|
||
|
|
// Update Bitwarden with new secret
|
||
|
|
var existingBwsId = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev));
|
||
|
|
if (!string.IsNullOrEmpty(existingBwsId))
|
||
|
|
{
|
||
|
|
await bws.UpdateSecretAsync(
|
||
|
|
existingBwsId,
|
||
|
|
$"{abbrev}/xibo-oauth-secret",
|
||
|
|
newSecret,
|
||
|
|
$"Xibo CMS OAuth2 client secret (rotated). ClientId: {newClientId}");
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
await bws.CreateInstanceSecretAsync(
|
||
|
|
key: $"{abbrev}/xibo-oauth-secret",
|
||
|
|
value: newSecret,
|
||
|
|
note: $"Xibo CMS OAuth2 client secret (rotated). ClientId: {newClientId}");
|
||
|
|
}
|
||
|
|
|
||
|
|
// Update OauthAppRegistry with new clientId
|
||
|
|
db.OauthAppRegistries.Add(new OauthAppRegistry
|
||
|
|
{
|
||
|
|
Id = Guid.NewGuid(),
|
||
|
|
InstanceId = ctx.InstanceId,
|
||
|
|
ClientId = newClientId,
|
||
|
|
CreatedAt = DateTime.UtcNow,
|
||
|
|
});
|
||
|
|
|
||
|
|
// Also update the settings cache with the new client ID
|
||
|
|
await settings.SetAsync(
|
||
|
|
SettingsService.InstanceOAuthClientId(abbrev),
|
||
|
|
newClientId,
|
||
|
|
SettingsService.CatInstance,
|
||
|
|
isSensitive: false);
|
||
|
|
|
||
|
|
await db.SaveChangesAsync(ct);
|
||
|
|
return $"Credentials stored. ClientId={newClientId} in OauthAppRegistry + Bitwarden.";
|
||
|
|
}, ct);
|
||
|
|
|
||
|
|
// ── Step 4: verify-access ───────────────────────────────────────
|
||
|
|
await runner.RunAsync("verify-access", async () =>
|
||
|
|
{
|
||
|
|
var verifyClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, newClientId, newSecret);
|
||
|
|
var aboutResp = await verifyClient.GetAboutAsync();
|
||
|
|
EnsureSuccess(aboutResp);
|
||
|
|
return $"New credentials verified. GET /about succeeded on {ctx.InstanceUrl}.";
|
||
|
|
}, ct);
|
||
|
|
|
||
|
|
// ── Step 5: audit-log ───────────────────────────────────────────
|
||
|
|
await runner.RunAsync("audit-log", async () =>
|
||
|
|
{
|
||
|
|
db.AuditLogs.Add(new AuditLog
|
||
|
|
{
|
||
|
|
Id = Guid.NewGuid(),
|
||
|
|
InstanceId = ctx.InstanceId,
|
||
|
|
Actor = "system/credential-rotation",
|
||
|
|
Action = "rotate-oauth2",
|
||
|
|
Target = $"xibo-{abbrev}",
|
||
|
|
Outcome = "success",
|
||
|
|
Detail = $"OAuth2 credentials rotated. Old clientId={currentClientId} → new clientId={newClientId}. Job {job.Id}.",
|
||
|
|
OccurredAt = DateTime.UtcNow,
|
||
|
|
});
|
||
|
|
await db.SaveChangesAsync(ct);
|
||
|
|
|
||
|
|
return $"AuditLog entry written. OAuth2 rotation complete.";
|
||
|
|
}, ct);
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
// CRITICAL: Steps 3-5 failed AFTER step 2 created the new app.
|
||
|
|
// The old credentials are already deleted. Log new credentials for manual recovery.
|
||
|
|
_logger.LogCritical(ex,
|
||
|
|
"CRITICAL — OAuth2 rotation for '{Abbrev}' failed after new app creation. " +
|
||
|
|
"Old clientId={OldClientId} (DELETED). " +
|
||
|
|
"New clientId={NewClientId}, New secret={NewSecret}. " +
|
||
|
|
"EMERGENCY RECOVERY DATA — store these credentials manually.",
|
||
|
|
abbrev, currentClientId, newClientId, newSecret);
|
||
|
|
|
||
|
|
// Also persist to a JobStep for operator visibility
|
||
|
|
var emergencyStep = new JobStep
|
||
|
|
{
|
||
|
|
Id = Guid.NewGuid(),
|
||
|
|
JobId = job.Id,
|
||
|
|
StepName = "emergency-credential-log",
|
||
|
|
Status = JobStepStatus.Failed,
|
||
|
|
StartedAt = DateTime.UtcNow,
|
||
|
|
CompletedAt = DateTime.UtcNow,
|
||
|
|
LogOutput = $"CRITICAL EMERGENCY RECOVERY DATA — OAuth2 rotation partial failure. " +
|
||
|
|
$"Old clientId={currentClientId} (DELETED). " +
|
||
|
|
$"New clientId={newClientId}. New secret={newSecret}. " +
|
||
|
|
$"These credentials must be stored manually. Error: {ex.Message}",
|
||
|
|
};
|
||
|
|
db.JobSteps.Add(emergencyStep);
|
||
|
|
await db.SaveChangesAsync(CancellationToken.None);
|
||
|
|
|
||
|
|
throw; // Re-throw to fail the job
|
||
|
|
}
|
||
|
|
|
||
|
|
_logger.LogInformation(
|
||
|
|
"RotateCredentialsPipeline completed for job {JobId} (abbrev={Abbrev}, newClientId={ClientId})",
|
||
|
|
job.Id, abbrev, newClientId);
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─────────────────────────────────────────────────────────────────────────
|
||
|
|
// Helpers
|
||
|
|
// ─────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
private static async Task<PipelineContext> BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct)
|
||
|
|
{
|
||
|
|
var customer = await db.Customers
|
||
|
|
.Include(c => c.Instances)
|
||
|
|
.FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct)
|
||
|
|
?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}.");
|
||
|
|
|
||
|
|
var instance = customer.Instances.FirstOrDefault()
|
||
|
|
?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}.");
|
||
|
|
|
||
|
|
var abbrev = customer.Abbreviation.ToLowerInvariant();
|
||
|
|
|
||
|
|
return new PipelineContext
|
||
|
|
{
|
||
|
|
JobId = job.Id,
|
||
|
|
CustomerId = customer.Id,
|
||
|
|
InstanceId = instance.Id,
|
||
|
|
Abbreviation = abbrev,
|
||
|
|
CompanyName = customer.CompanyName,
|
||
|
|
AdminEmail = customer.AdminEmail,
|
||
|
|
AdminFirstName = customer.AdminFirstName,
|
||
|
|
InstanceUrl = instance.XiboUrl,
|
||
|
|
DockerStackName = instance.DockerStackName,
|
||
|
|
ParametersJson = job.Parameters,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
private static void EnsureSuccess<T>(Refit.IApiResponse<T> response)
|
||
|
|
{
|
||
|
|
if (!response.IsSuccessStatusCode)
|
||
|
|
throw new InvalidOperationException(
|
||
|
|
$"API call failed with status {response.StatusCode}: {response.Error?.Content}");
|
||
|
|
}
|
||
|
|
}
|