Files
OTSSignsOrchestrator/OTSSignsOrchestrator.Desktop/ViewModels/InstancesViewModel.cs
Matt Batchelder c6d46098dd feat: Implement provisioning pipelines for subscription management
- Add ReactivatePipeline to handle subscription reactivation, including scaling Docker services, health verification, status updates, audit logging, and broadcasting status changes.
- Introduce RotateCredentialsPipeline for OAuth2 credential rotation, managing the deletion of old apps, creation of new ones, credential storage, access verification, and audit logging.
- Create StepRunner to manage job step execution, including lifecycle management and progress broadcasting via SignalR.
- Implement SuspendPipeline for subscription suspension, scaling down services, updating statuses, logging audits, and broadcasting changes.
- Add UpdateScreenLimitPipeline to update Xibo CMS screen limits and record snapshots.
- Introduce XiboFeatureManifests for hardcoded feature ACLs per role.
- Add docker-compose.dev.yml for local development with PostgreSQL setup.
2026-03-18 10:27:26 -04:00

628 lines
26 KiB
C#

using System.Collections.ObjectModel;
using Avalonia.Threading;
using CommunityToolkit.Mvvm.ComponentModel;
using CommunityToolkit.Mvvm.Input;
using CommunityToolkit.Mvvm.Messaging;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using OTSSignsOrchestrator.Core.Data;
using OTSSignsOrchestrator.Core.Models.DTOs;
using OTSSignsOrchestrator.Core.Models.Entities;
using OTSSignsOrchestrator.Core.Services;
using OTSSignsOrchestrator.Desktop.Models;
using OTSSignsOrchestrator.Desktop.Services;
namespace OTSSignsOrchestrator.Desktop.ViewModels;
/// <summary>
/// ViewModel for listing, viewing, and managing CMS instances.
/// All data is fetched live from Docker Swarm hosts — nothing stored locally.
/// Server operations (decommission, suspend, reactivate) go through the REST API.
/// Real-time updates arrive via SignalR → WeakReferenceMessenger.
/// </summary>
public partial class InstancesViewModel : ObservableObject,
IRecipient<AlertRaisedMessage>,
IRecipient<InstanceStatusChangedMessage>,
IRecipient<JobCreatedMessage>,
IRecipient<JobCompletedMessage>
{
private readonly IServiceProvider _services;
private readonly ILogger<InstancesViewModel> _logger;
private readonly IServerApiClient? _serverApi;
[ObservableProperty] private ObservableCollection<LiveStackItem> _instances = new();
[ObservableProperty] private LiveStackItem? _selectedInstance;
[ObservableProperty] private string _statusMessage = string.Empty;
[ObservableProperty] private bool _isBusy;
[ObservableProperty] private string _filterText = string.Empty;
[ObservableProperty] private ObservableCollection<ServiceInfo> _selectedServices = new();
// Available SSH hosts — loaded for display and used to scope operations
[ObservableProperty] private ObservableCollection<SshHost> _availableHosts = new();
[ObservableProperty] private SshHost? _selectedSshHost;
// ── P1 Authentik Banner ──────────────────────────────────────────────────
[ObservableProperty] private bool _isAuthentikP1BannerVisible;
[ObservableProperty] private string _authentikP1Message = string.Empty;
// ── Container Logs ──────────────────────────────────────────────────────
[ObservableProperty] private ObservableCollection<ServiceLogEntry> _logEntries = new();
[ObservableProperty] private ObservableCollection<string> _logServiceFilter = new();
[ObservableProperty] private string _selectedLogService = "All Services";
[ObservableProperty] private bool _isLogsPanelVisible;
[ObservableProperty] private bool _isLogsAutoRefresh = true;
[ObservableProperty] private bool _isLoadingLogs;
[ObservableProperty] private string _logsStatusMessage = string.Empty;
[ObservableProperty] private int _logTailLines = 200;
private DispatcherTimer? _logRefreshTimer;
private bool _isLogRefreshRunning;
/// <summary>Raised when the instance details modal should be opened for the given ViewModel.</summary>
public event Action<InstanceDetailsViewModel>? OpenDetailsRequested;
/// <summary>
/// Callback the View wires up to show a confirmation dialog.
/// Parameters: (title, message) → returns true if the user confirmed.
/// </summary>
public Func<string, string, Task<bool>>? ConfirmAsync { get; set; }
/// <summary>
/// Callback the View wires up to show a multi-step confirmation dialog for decommission.
/// Parameters: (abbreviation) → returns true if confirmed through all steps.
/// </summary>
public Func<string, Task<bool>>? ConfirmDecommissionAsync { get; set; }
private string? _pendingSelectAbbrev;
public InstancesViewModel(IServiceProvider services)
{
_services = services;
_logger = services.GetRequiredService<ILogger<InstancesViewModel>>();
_serverApi = services.GetService<IServerApiClient>();
// Register for SignalR messages via WeakReferenceMessenger
WeakReferenceMessenger.Default.Register<AlertRaisedMessage>(this);
WeakReferenceMessenger.Default.Register<InstanceStatusChangedMessage>(this);
WeakReferenceMessenger.Default.Register<JobCreatedMessage>(this);
WeakReferenceMessenger.Default.Register<JobCompletedMessage>(this);
_ = RefreshAllAsync();
}
/// <summary>
/// Queues an abbreviation to be auto-selected once the next live refresh completes.
/// Call immediately after construction (before <see cref="RefreshAllAsync"/> finishes).
/// </summary>
public void SetPendingSelection(string abbrev)
=> _pendingSelectAbbrev = abbrev;
// ── SignalR Message Handlers ─────────────────────────────────────────────
// These are called on the UI thread (SignalR handlers dispatch via Dispatcher.UIThread).
void IRecipient<AlertRaisedMessage>.Receive(AlertRaisedMessage message)
{
var (severity, msg) = message.Value;
if (string.Equals(severity, "Critical", StringComparison.OrdinalIgnoreCase) &&
msg.Contains("Authentik", StringComparison.OrdinalIgnoreCase))
{
AuthentikP1Message = msg;
IsAuthentikP1BannerVisible = true;
}
}
void IRecipient<InstanceStatusChangedMessage>.Receive(InstanceStatusChangedMessage message)
{
var (customerId, status) = message.Value;
_logger.LogInformation("Instance status changed: customer={CustomerId} status={Status}", customerId, status);
StatusMessage = $"Instance {customerId} status → {status}";
// Refresh the list to pick up the new status
_ = RefreshAllAsync();
}
void IRecipient<JobCreatedMessage>.Receive(JobCreatedMessage message)
{
var (jobId, abbrev, jobType) = message.Value;
_logger.LogInformation("Job created: {JobId} type={JobType} abbrev={Abbrev}", jobId, jobType, abbrev);
StatusMessage = $"Job '{jobType}' created for {abbrev} (id: {jobId[..8]}…)";
}
void IRecipient<JobCompletedMessage>.Receive(JobCompletedMessage message)
{
var (jobId, success, summary) = message.Value;
_logger.LogInformation("Job completed: {JobId} success={Success} summary={Summary}", jobId, success, summary);
StatusMessage = success
? $"Job {jobId[..8]}… completed: {summary}"
: $"Job {jobId[..8]}… failed: {summary}";
// Refresh the instance list to reflect changes from the completed job
_ = RefreshAllAsync();
}
// ── Load / Refresh ──────────────────────────────────────────────────────
/// <summary>
/// Enumerates all SSH hosts, then calls docker stack ls on each to build the
/// live instance list. Only stacks matching *-cms-stack are shown.
/// </summary>
[RelayCommand]
private async Task LoadInstancesAsync() => await RefreshAllAsync();
private async Task RefreshAllAsync()
{
IsBusy = true;
StatusMessage = "Loading live instances from all hosts...";
SelectedServices = new ObservableCollection<ServiceInfo>();
try
{
using var scope = _services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<XiboContext>();
var hosts = await db.SshHosts.OrderBy(h => h.Label).ToListAsync();
AvailableHosts = new ObservableCollection<SshHost>(hosts);
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
var all = new List<LiveStackItem>();
var errors = new List<string>();
foreach (var host in hosts)
{
try
{
dockerCli.SetHost(host);
var stacks = await dockerCli.ListStacksAsync();
foreach (var stack in stacks.Where(s => s.Name.EndsWith("-cms-stack")))
{
all.Add(new LiveStackItem
{
StackName = stack.Name,
CustomerAbbrev = stack.Name[..^10],
ServiceCount = stack.ServiceCount,
Host = host,
});
}
}
catch (Exception ex) { errors.Add($"{host.Label}: {ex.Message}"); }
}
// Enrich with server-side customer IDs if the server API is available
if (_serverApi is not null)
{
try
{
var fleet = await _serverApi.GetFleetAsync();
var lookup = fleet.ToDictionary(f => f.Abbreviation, f => f.CustomerId, StringComparer.OrdinalIgnoreCase);
foreach (var item in all)
{
if (lookup.TryGetValue(item.CustomerAbbrev, out var customerId))
item.CustomerId = customerId;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not enrich instances with server fleet data");
}
}
if (!string.IsNullOrWhiteSpace(FilterText))
all = all.Where(i =>
i.StackName.Contains(FilterText, StringComparison.OrdinalIgnoreCase) ||
i.CustomerAbbrev.Contains(FilterText, StringComparison.OrdinalIgnoreCase) ||
i.HostLabel.Contains(FilterText, StringComparison.OrdinalIgnoreCase)).ToList();
Instances = new ObservableCollection<LiveStackItem>(all);
// Auto-select a pending instance (e.g. just deployed from Create Instance page)
if (_pendingSelectAbbrev is not null)
{
SelectedInstance = all.FirstOrDefault(i =>
i.CustomerAbbrev.Equals(_pendingSelectAbbrev, StringComparison.OrdinalIgnoreCase));
_pendingSelectAbbrev = null;
}
var msg = $"Found {all.Count} instance(s) across {hosts.Count} host(s).";
if (errors.Count > 0) msg += $" | Errors: {string.Join(" | ", errors)}";
StatusMessage = msg;
}
catch (Exception ex) { StatusMessage = $"Error: {ex.Message}"; }
finally { IsBusy = false; }
}
[RelayCommand]
private async Task InspectInstanceAsync()
{
if (SelectedInstance == null) return;
IsBusy = true;
StatusMessage = $"Inspecting '{SelectedInstance.StackName}'...";
try
{
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
var services = await dockerCli.InspectStackServicesAsync(SelectedInstance.StackName);
SelectedServices = new ObservableCollection<ServiceInfo>(services);
StatusMessage = $"Found {services.Count} service(s) in stack '{SelectedInstance.StackName}'.";
// Populate service filter dropdown and show logs panel
var filterItems = new List<string> { "All Services" };
filterItems.AddRange(services.Select(s => s.Name));
LogServiceFilter = new ObservableCollection<string>(filterItems);
SelectedLogService = "All Services";
IsLogsPanelVisible = true;
// Fetch initial logs and start auto-refresh
await FetchLogsInternalAsync();
StartLogAutoRefresh();
}
catch (Exception ex) { StatusMessage = $"Error inspecting: {ex.Message}"; }
finally { IsBusy = false; }
}
// ── Container Log Commands ──────────────────────────────────────────────
[RelayCommand]
private async Task RefreshLogsAsync()
{
await FetchLogsInternalAsync();
}
[RelayCommand]
private void ToggleLogsAutoRefresh()
{
IsLogsAutoRefresh = !IsLogsAutoRefresh;
if (IsLogsAutoRefresh)
StartLogAutoRefresh();
else
StopLogAutoRefresh();
}
[RelayCommand]
private void CloseLogsPanel()
{
StopLogAutoRefresh();
IsLogsPanelVisible = false;
LogEntries = new ObservableCollection<ServiceLogEntry>();
LogsStatusMessage = string.Empty;
}
partial void OnSelectedLogServiceChanged(string value)
{
// When user changes the service filter, refresh logs immediately
if (IsLogsPanelVisible)
_ = FetchLogsInternalAsync();
}
private async Task FetchLogsInternalAsync()
{
if (SelectedInstance == null || _isLogRefreshRunning) return;
_isLogRefreshRunning = true;
IsLoadingLogs = true;
try
{
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
string? serviceFilter = SelectedLogService == "All Services" ? null : SelectedLogService;
var entries = await dockerCli.GetServiceLogsAsync(
SelectedInstance.StackName, serviceFilter, LogTailLines);
LogEntries = new ObservableCollection<ServiceLogEntry>(entries);
LogsStatusMessage = $"{entries.Count} log line(s) · last fetched {DateTime.Now:HH:mm:ss}";
}
catch (Exception ex)
{
LogsStatusMessage = $"Error fetching logs: {ex.Message}";
}
finally
{
IsLoadingLogs = false;
_isLogRefreshRunning = false;
}
}
private void StartLogAutoRefresh()
{
StopLogAutoRefresh();
if (!IsLogsAutoRefresh) return;
_logRefreshTimer = new DispatcherTimer
{
Interval = TimeSpan.FromSeconds(5)
};
_logRefreshTimer.Tick += async (_, _) =>
{
if (IsLogsPanelVisible && IsLogsAutoRefresh && !_isLogRefreshRunning)
await FetchLogsInternalAsync();
};
_logRefreshTimer.Start();
}
private void StopLogAutoRefresh()
{
_logRefreshTimer?.Stop();
_logRefreshTimer = null;
}
// ── Restart Commands ────────────────────────────────────────────────
[RelayCommand]
private async Task RestartStackAsync()
{
if (SelectedInstance == null) return;
if (ConfirmAsync is not null)
{
var confirmed = await ConfirmAsync(
"Restart Stack",
$"Are you sure you want to restart all services in '{SelectedInstance.StackName}'?\n\nThis will force-update every service in the stack, causing brief downtime.");
if (!confirmed) return;
}
IsBusy = true;
StatusMessage = $"Restarting all services in '{SelectedInstance.StackName}'...";
try
{
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
var services = await dockerCli.InspectStackServicesAsync(SelectedInstance.StackName);
var failures = new List<string>();
for (var i = 0; i < services.Count; i++)
{
var svc = services[i];
StatusMessage = $"Restarting service {i + 1}/{services.Count}: {svc.Name}...";
var ok = await dockerCli.ForceUpdateServiceAsync(svc.Name);
if (!ok) failures.Add(svc.Name);
}
StatusMessage = failures.Count == 0
? $"All {services.Count} service(s) in '{SelectedInstance.StackName}' restarted successfully."
: $"Restarted with errors — failed services: {string.Join(", ", failures)}";
}
catch (Exception ex) { StatusMessage = $"Error restarting stack: {ex.Message}"; }
finally { IsBusy = false; }
}
[RelayCommand]
private async Task RestartServiceAsync(ServiceInfo? service)
{
if (service is null || SelectedInstance is null) return;
if (ConfirmAsync is not null)
{
var confirmed = await ConfirmAsync(
"Restart Service",
$"Are you sure you want to restart '{service.Name}'?\n\nThis will force-update the service, causing its tasks to be recreated.");
if (!confirmed) return;
}
IsBusy = true;
StatusMessage = $"Restarting service '{service.Name}'...";
try
{
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
var ok = await dockerCli.ForceUpdateServiceAsync(service.Name);
StatusMessage = ok
? $"Service '{service.Name}' restarted successfully."
: $"Failed to restart service '{service.Name}'.";
// Refresh services to show updated replica status
var services = await dockerCli.InspectStackServicesAsync(SelectedInstance.StackName);
SelectedServices = new ObservableCollection<ServiceInfo>(services);
}
catch (Exception ex) { StatusMessage = $"Error restarting service: {ex.Message}"; }
finally { IsBusy = false; }
}
[RelayCommand]
private async Task DeleteInstanceAsync()
{
if (SelectedInstance == null) return;
IsBusy = true;
StatusMessage = $"Deleting {SelectedInstance.StackName}...";
try
{
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
var dockerSecrets = _services.GetRequiredService<SshDockerSecretsService>();
dockerSecrets.SetHost(SelectedInstance.Host);
using var scope = _services.CreateScope();
var instanceSvc = scope.ServiceProvider.GetRequiredService<InstanceService>();
var result = await instanceSvc.DeleteInstanceAsync(
SelectedInstance.StackName, SelectedInstance.CustomerAbbrev);
StatusMessage = result.Success
? $"Instance '{SelectedInstance.StackName}' deleted."
: $"Delete failed: {result.ErrorMessage}";
await RefreshAllAsync();
}
catch (Exception ex) { StatusMessage = $"Error deleting: {ex.Message}"; }
finally { IsBusy = false; }
}
[RelayCommand]
private async Task RotateMySqlPasswordAsync()
{
if (SelectedInstance == null) return;
IsBusy = true;
StatusMessage = $"Rotating MySQL password for {SelectedInstance.StackName}...";
try
{
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
var dockerSecrets = _services.GetRequiredService<SshDockerSecretsService>();
dockerSecrets.SetHost(SelectedInstance.Host);
using var scope = _services.CreateScope();
var instanceSvc = scope.ServiceProvider.GetRequiredService<InstanceService>();
var (ok, msg) = await instanceSvc.RotateMySqlPasswordAsync(SelectedInstance.StackName);
StatusMessage = ok ? $"Done: {msg}" : $"Failed: {msg}";
await RefreshAllAsync();
}
catch (Exception ex) { StatusMessage = $"Error rotating password: {ex.Message}"; }
finally { IsBusy = false; }
}
// ── P1 Banner Commands ────────────────────────────────────────────────
[RelayCommand]
private void DismissP1Banner()
{
IsAuthentikP1BannerVisible = false;
AuthentikP1Message = string.Empty;
}
/// <summary>
/// Called from a SignalR <c>AlertRaised</c> handler (runs on a background thread).
/// CRITICAL: wraps all property updates with <see cref="Dispatcher.UIThread"/> to
/// avoid silent cross-thread exceptions in Avalonia.
/// </summary>
public void HandleAlertRaised(string severity, string message)
{
if (string.Equals(severity, "Critical", StringComparison.OrdinalIgnoreCase) &&
message.Contains("Authentik", StringComparison.OrdinalIgnoreCase))
{
Dispatcher.UIThread.InvokeAsync(() =>
{
AuthentikP1Message = message;
IsAuthentikP1BannerVisible = true;
});
}
}
// ── Server-side Job Commands (decommission, suspend, reactivate) ────────
// Desktop has NO direct infrastructure access — all operations go through the server REST API.
[RelayCommand]
private async Task DecommissionAsync(LiveStackItem? instance)
{
instance ??= SelectedInstance;
if (instance is null) return;
if (instance.CustomerId is null)
{
StatusMessage = "Cannot decommission: no server-side customer ID available for this instance.";
return;
}
// Multi-step confirmation: user must type the abbreviation to confirm
if (ConfirmDecommissionAsync is not null)
{
var confirmed = await ConfirmDecommissionAsync(instance.CustomerAbbrev);
if (!confirmed) return;
}
else if (ConfirmAsync is not null)
{
var confirmed = await ConfirmAsync(
"Decommission Instance",
$"Are you sure you want to decommission '{instance.CustomerAbbrev}'?\n\n" +
"This will:\n" +
" • Remove all Docker services and stack\n" +
" • Delete Docker secrets\n" +
" • Remove NFS volumes and data\n" +
" • Revoke Authentik provider\n" +
" • Mark the customer as decommissioned\n\n" +
"This action is IRREVERSIBLE.");
if (!confirmed) return;
}
await CreateServerJobAsync(instance, "decommission");
}
[RelayCommand]
private async Task SuspendInstanceAsync(LiveStackItem? instance)
{
instance ??= SelectedInstance;
if (instance is null) return;
if (instance.CustomerId is null)
{
StatusMessage = "Cannot suspend: no server-side customer ID available for this instance.";
return;
}
if (ConfirmAsync is not null)
{
var confirmed = await ConfirmAsync(
"Suspend Instance",
$"Are you sure you want to suspend '{instance.CustomerAbbrev}'?\n\n" +
"The instance will be scaled to zero replicas. Data will be preserved.");
if (!confirmed) return;
}
await CreateServerJobAsync(instance, "suspend");
}
[RelayCommand]
private async Task ReactivateInstanceAsync(LiveStackItem? instance)
{
instance ??= SelectedInstance;
if (instance is null) return;
if (instance.CustomerId is null)
{
StatusMessage = "Cannot reactivate: no server-side customer ID available for this instance.";
return;
}
await CreateServerJobAsync(instance, "reactivate");
}
private async Task CreateServerJobAsync(LiveStackItem instance, string jobType)
{
if (_serverApi is null)
{
StatusMessage = "Server API client is not configured.";
return;
}
IsBusy = true;
StatusMessage = $"Requesting '{jobType}' for {instance.CustomerAbbrev}...";
try
{
var response = await _serverApi.CreateJobAsync(
new CreateJobRequest(instance.CustomerId!.Value, jobType, null));
StatusMessage = $"Job '{jobType}' created (id: {response.Id.ToString()[..8]}…). Status: {response.Status}";
_logger.LogInformation("Server job created: {JobId} type={JobType} customer={CustomerId}",
response.Id, jobType, instance.CustomerId);
}
catch (Refit.ApiException ex)
{
StatusMessage = $"Server error creating '{jobType}' job: {ex.StatusCode} — {ex.Content}";
_logger.LogError(ex, "Failed to create {JobType} job for customer {CustomerId}", jobType, instance.CustomerId);
}
catch (Exception ex)
{
StatusMessage = $"Error creating '{jobType}' job: {ex.Message}";
_logger.LogError(ex, "Failed to create {JobType} job for customer {CustomerId}", jobType, instance.CustomerId);
}
finally { IsBusy = false; }
}
// ── Details ─────────────────────────────────────────────────────────────
[RelayCommand]
private async Task OpenDetailsAsync()
{
if (SelectedInstance == null) return;
IsBusy = true;
StatusMessage = $"Loading details for '{SelectedInstance.StackName}'...";
try
{
// Set the SSH host on singleton Docker services so modal operations target the right host
var dockerCli = _services.GetRequiredService<SshDockerCliService>();
dockerCli.SetHost(SelectedInstance.Host);
var dockerSecrets = _services.GetRequiredService<SshDockerSecretsService>();
dockerSecrets.SetHost(SelectedInstance.Host);
var detailsVm = _services.GetRequiredService<InstanceDetailsViewModel>();
await detailsVm.LoadAsync(SelectedInstance);
OpenDetailsRequested?.Invoke(detailsVm);
StatusMessage = string.Empty;
}
catch (Exception ex) { StatusMessage = $"Error opening details: {ex.Message}"; }
finally { IsBusy = false; }
}
}