using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using OTSSignsOrchestrator.Core.Services; using OTSSignsOrchestrator.Server.Clients; using OTSSignsOrchestrator.Server.Data; using OTSSignsOrchestrator.Server.Data.Entities; using OTSSignsOrchestrator.Server.Hubs; namespace OTSSignsOrchestrator.Server.Workers; /// /// OAuth2 credential rotation pipeline — deletes the old Xibo OAuth app, creates a new one, /// stores the new credentials, and verifies access. Handles JobType = "rotate-oauth2". /// /// CRITICAL: OAuth2 clientId CHANGES on rotation — there is no in-place secret refresh. /// The secret is returned ONLY in the POST /api/application response. /// /// Steps: /// 1. delete-old-app — Delete current OAuth2 application via Xibo API /// 2. create-new-app — POST /api/application — IMMEDIATELY capture secret /// 3. store-credentials — Update Bitwarden + OauthAppRegistry with new clientId /// 4. verify-access — Test new credentials via client_credentials flow /// 5. audit-log — Write AuditLog (or CRITICAL error with emergency creds if steps 3-4 fail) /// public sealed class RotateCredentialsPipeline : IProvisioningPipeline { public string HandlesJobType => "rotate-oauth2"; private const int TotalSteps = 5; private readonly IServiceProvider _services; private readonly ILogger _logger; public RotateCredentialsPipeline( IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } public async Task ExecuteAsync(Job job, CancellationToken ct) { await using var scope = _services.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); var hub = scope.ServiceProvider.GetRequiredService>(); var xiboFactory = scope.ServiceProvider.GetRequiredService(); var bws = scope.ServiceProvider.GetRequiredService(); var settings = scope.ServiceProvider.GetRequiredService(); var ctx = await BuildContextAsync(job, db, ct); var runner = new StepRunner(db, hub, _logger, job.Id, TotalSteps); var abbrev = ctx.Abbreviation; // Load current OAuth2 credentials var currentReg = await db.OauthAppRegistries .Where(r => r.InstanceId == ctx.InstanceId) .OrderByDescending(r => r.CreatedAt) .FirstOrDefaultAsync(ct) ?? throw new InvalidOperationException( $"No OauthAppRegistry found for instance {ctx.InstanceId}. Cannot rotate."); var currentSecret = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev)) ?? throw new InvalidOperationException( $"OAuth secret not found in settings for instance '{abbrev}'. Cannot rotate."); var currentClientId = currentReg.ClientId; // Get a Xibo API client using the current (about-to-be-deleted) credentials var xiboClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, currentClientId, currentSecret); // Mutable state — captured across steps string newClientId = string.Empty; string newSecret = string.Empty; // ── Step 1: delete-old-app ────────────────────────────────────────── await runner.RunAsync("delete-old-app", async () => { await xiboClient.DeleteApplicationAsync(currentClientId); return $"Old OAuth2 application deleted. ClientId={currentClientId}."; }, ct); // ── Step 2: create-new-app ────────────────────────────────────────── // CRITICAL: After this step, the old credentials are gone. If subsequent steps fail, // the new clientId + secret MUST be logged for manual recovery. await runner.RunAsync("create-new-app", async () => { // Need a client with some auth to create the new app. Use the instance's bootstrap // admin credentials (ots-admin user) via direct login if available, or we can // re-authenticate using the base URL. Since the old app is deleted, we need a // different auth mechanism. The Xibo CMS allows creating apps as any authenticated // super-admin. Use the ots-admin password from Bitwarden. // // However, the simplest path: the DELETE above invalidated the old token, but the // Xibo CMS still has the ots-admin and ots-svc users. We stored the admin password // in Bitwarden. Retrieve it and create a new factory client. var adminPassBwsId = await settings.GetAsync(SettingsService.InstanceAdminPasswordSecretId(abbrev)); if (string.IsNullOrEmpty(adminPassBwsId)) throw new InvalidOperationException( $"Admin password Bitwarden secret ID not found for '{abbrev}'. Cannot create new OAuth app."); var adminPassSecret = await bws.GetSecretAsync(adminPassBwsId); var bootstrapClientId = await settings.GetAsync(SettingsService.XiboBootstrapClientId) ?? throw new InvalidOperationException("Bootstrap OAuth client ID not configured."); var bootstrapClientSecret = await settings.GetAsync(SettingsService.XiboBootstrapClientSecret) ?? throw new InvalidOperationException("Bootstrap OAuth client secret not configured."); // Re-create client using bootstrap credentials var bootstrapClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, bootstrapClientId, bootstrapClientSecret); var appName = $"OTS Orchestrator — {abbrev.ToUpperInvariant()}"; var response = await bootstrapClient.CreateApplicationAsync(new CreateApplicationRequest(appName)); EnsureSuccess(response); var data = response.Content ?? throw new InvalidOperationException("CreateApplication returned empty response."); // CRITICAL: Capture secret IMMEDIATELY — it is returned ONLY in this response. newClientId = data["key"]?.ToString() ?? throw new InvalidOperationException("OAuth application response missing 'key'."); newSecret = data["secret"]?.ToString() ?? throw new InvalidOperationException("OAuth application response missing 'secret'."); return $"New OAuth2 application created. ClientId={newClientId}. Secret captured (stored in next step)."; }, ct); // Steps 3-5: if ANY of these fail after step 2, we must log the credentials for recovery. // The old app is already deleted — there is no rollback path. try { // ── Step 3: store-credentials ─────────────────────────────────── await runner.RunAsync("store-credentials", async () => { // Update Bitwarden with new secret var existingBwsId = await settings.GetAsync(SettingsService.InstanceOAuthSecretId(abbrev)); if (!string.IsNullOrEmpty(existingBwsId)) { await bws.UpdateSecretAsync( existingBwsId, $"{abbrev}/xibo-oauth-secret", newSecret, $"Xibo CMS OAuth2 client secret (rotated). ClientId: {newClientId}"); } else { await bws.CreateInstanceSecretAsync( key: $"{abbrev}/xibo-oauth-secret", value: newSecret, note: $"Xibo CMS OAuth2 client secret (rotated). ClientId: {newClientId}"); } // Update OauthAppRegistry with new clientId db.OauthAppRegistries.Add(new OauthAppRegistry { Id = Guid.NewGuid(), InstanceId = ctx.InstanceId, ClientId = newClientId, CreatedAt = DateTime.UtcNow, }); // Also update the settings cache with the new client ID await settings.SetAsync( SettingsService.InstanceOAuthClientId(abbrev), newClientId, SettingsService.CatInstance, isSensitive: false); await db.SaveChangesAsync(ct); return $"Credentials stored. ClientId={newClientId} in OauthAppRegistry + Bitwarden."; }, ct); // ── Step 4: verify-access ─────────────────────────────────────── await runner.RunAsync("verify-access", async () => { var verifyClient = await xiboFactory.CreateAsync(ctx.InstanceUrl, newClientId, newSecret); var aboutResp = await verifyClient.GetAboutAsync(); EnsureSuccess(aboutResp); return $"New credentials verified. GET /about succeeded on {ctx.InstanceUrl}."; }, ct); // ── Step 5: audit-log ─────────────────────────────────────────── await runner.RunAsync("audit-log", async () => { db.AuditLogs.Add(new AuditLog { Id = Guid.NewGuid(), InstanceId = ctx.InstanceId, Actor = "system/credential-rotation", Action = "rotate-oauth2", Target = $"xibo-{abbrev}", Outcome = "success", Detail = $"OAuth2 credentials rotated. Old clientId={currentClientId} → new clientId={newClientId}. Job {job.Id}.", OccurredAt = DateTime.UtcNow, }); await db.SaveChangesAsync(ct); return $"AuditLog entry written. OAuth2 rotation complete."; }, ct); } catch (Exception ex) { // CRITICAL: Steps 3-5 failed AFTER step 2 created the new app. // The old credentials are already deleted. Log new credentials for manual recovery. _logger.LogCritical(ex, "CRITICAL — OAuth2 rotation for '{Abbrev}' failed after new app creation. " + "Old clientId={OldClientId} (DELETED). " + "New clientId={NewClientId}, New secret={NewSecret}. " + "EMERGENCY RECOVERY DATA — store these credentials manually.", abbrev, currentClientId, newClientId, newSecret); // Also persist to a JobStep for operator visibility var emergencyStep = new JobStep { Id = Guid.NewGuid(), JobId = job.Id, StepName = "emergency-credential-log", Status = JobStepStatus.Failed, StartedAt = DateTime.UtcNow, CompletedAt = DateTime.UtcNow, LogOutput = $"CRITICAL EMERGENCY RECOVERY DATA — OAuth2 rotation partial failure. " + $"Old clientId={currentClientId} (DELETED). " + $"New clientId={newClientId}. New secret={newSecret}. " + $"These credentials must be stored manually. Error: {ex.Message}", }; db.JobSteps.Add(emergencyStep); await db.SaveChangesAsync(CancellationToken.None); throw; // Re-throw to fail the job } _logger.LogInformation( "RotateCredentialsPipeline completed for job {JobId} (abbrev={Abbrev}, newClientId={ClientId})", job.Id, abbrev, newClientId); } // ───────────────────────────────────────────────────────────────────────── // Helpers // ───────────────────────────────────────────────────────────────────────── private static async Task BuildContextAsync(Job job, OrchestratorDbContext db, CancellationToken ct) { var customer = await db.Customers .Include(c => c.Instances) .FirstOrDefaultAsync(c => c.Id == job.CustomerId, ct) ?? throw new InvalidOperationException($"Customer {job.CustomerId} not found for job {job.Id}."); var instance = customer.Instances.FirstOrDefault() ?? throw new InvalidOperationException($"No instance found for customer {job.CustomerId}."); var abbrev = customer.Abbreviation.ToLowerInvariant(); return new PipelineContext { JobId = job.Id, CustomerId = customer.Id, InstanceId = instance.Id, Abbreviation = abbrev, CompanyName = customer.CompanyName, AdminEmail = customer.AdminEmail, AdminFirstName = customer.AdminFirstName, InstanceUrl = instance.XiboUrl, DockerStackName = instance.DockerStackName, ParametersJson = job.Parameters, }; } private static void EnsureSuccess(Refit.IApiResponse response) { if (!response.IsSuccessStatusCode) throw new InvalidOperationException( $"API call failed with status {response.StatusCode}: {response.Error?.Content}"); } }