diff --git a/app/Console/Commands/TenantpilotBackfillFindingLifecycle.php b/app/Console/Commands/TenantpilotBackfillFindingLifecycle.php index ef3a625..7769fb6 100644 --- a/app/Console/Commands/TenantpilotBackfillFindingLifecycle.php +++ b/app/Console/Commands/TenantpilotBackfillFindingLifecycle.php @@ -4,10 +4,11 @@ namespace App\Console\Commands; -use App\Jobs\BackfillFindingLifecycleJob; use App\Models\Tenant; -use App\Services\OperationRunService; +use App\Services\Runbooks\FindingsLifecycleBackfillRunbookService; +use App\Services\Runbooks\FindingsLifecycleBackfillScope; use Illuminate\Console\Command; +use Illuminate\Validation\ValidationException; class TenantpilotBackfillFindingLifecycle extends Command { @@ -16,7 +17,7 @@ class TenantpilotBackfillFindingLifecycle extends Command protected $description = 'Queue tenant-scoped findings lifecycle backfill jobs idempotently.'; - public function handle(OperationRunService $operationRuns): int + public function handle(FindingsLifecycleBackfillRunbookService $runbookService): int { $tenantIdentifiers = array_values(array_filter((array) $this->option('tenant'))); @@ -36,25 +37,37 @@ public function handle(OperationRunService $operationRuns): int $queued = 0; $skipped = 0; + $nothingToDo = 0; foreach ($tenants as $tenant) { if (! $tenant instanceof Tenant) { continue; } - $run = $operationRuns->ensureRunWithIdentity( - tenant: $tenant, - type: 'findings.lifecycle.backfill', - identityInputs: [ - 'tenant_id' => (int) $tenant->getKey(), - 'trigger' => 'backfill', - ], - context: [ - 'workspace_id' => (int) $tenant->workspace_id, - 'source' => 'tenantpilot:findings:backfill-lifecycle', - ], - initiator: null, - ); + try { + $run = $runbookService->start( + scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), + initiator: null, + reason: null, + source: 'cli', + ); + } catch (ValidationException $e) { + $errors = $e->errors(); + + if (isset($errors['preflight.affected_count'])) { + $nothingToDo++; + + continue; + } + + $this->error(sprintf( + 'Backfill blocked for tenant %d: %s', + (int) $tenant->getKey(), + $e->getMessage(), + )); + + return self::FAILURE; + } if (! $run->wasRecentlyCreated) { $skipped++; @@ -62,21 +75,14 @@ public function handle(OperationRunService $operationRuns): int continue; } - $operationRuns->dispatchOrFail($run, function () use ($tenant): void { - BackfillFindingLifecycleJob::dispatch( - tenantId: (int) $tenant->getKey(), - workspaceId: (int) $tenant->workspace_id, - initiatorUserId: null, - ); - }); - $queued++; } $this->info(sprintf( - 'Queued %d backfill run(s), skipped %d duplicate run(s).', + 'Queued %d backfill run(s), skipped %d duplicate run(s), nothing to do %d.', $queued, $skipped, + $nothingToDo, )); return self::SUCCESS; diff --git a/app/Console/Commands/TenantpilotRunDeployRunbooks.php b/app/Console/Commands/TenantpilotRunDeployRunbooks.php new file mode 100644 index 0000000..fbc3428 --- /dev/null +++ b/app/Console/Commands/TenantpilotRunDeployRunbooks.php @@ -0,0 +1,51 @@ +start( + scope: FindingsLifecycleBackfillScope::allTenants(), + initiator: null, + reason: new RunbookReason( + reasonCode: RunbookReason::CODE_DATA_REPAIR, + reasonText: 'Deploy hook automated runbooks', + ), + source: 'deploy_hook', + ); + + $this->info('Deploy runbooks started (if needed).'); + + return self::SUCCESS; + } catch (ValidationException $e) { + $errors = $e->errors(); + + $skippable = isset($errors['preflight.affected_count']) || isset($errors['scope']); + + if ($skippable) { + $this->info('Deploy runbooks skipped (nothing to do or already running).'); + + return self::SUCCESS; + } + + $this->error('Deploy runbooks blocked by validation errors.'); + + return self::FAILURE; + } + } +} diff --git a/app/Filament/Resources/FindingResource/Pages/ListFindings.php b/app/Filament/Resources/FindingResource/Pages/ListFindings.php index 14329c2..658abe3 100644 --- a/app/Filament/Resources/FindingResource/Pages/ListFindings.php +++ b/app/Filament/Resources/FindingResource/Pages/ListFindings.php @@ -29,8 +29,10 @@ class ListFindings extends ListRecords protected function getHeaderActions(): array { - return [ - UiEnforcement::forAction( + $actions = []; + + if ((bool) config('tenantpilot.allow_admin_maintenance_actions', false)) { + $actions[] = UiEnforcement::forAction( Actions\Action::make('backfill_lifecycle') ->label('Backfill findings lifecycle') ->icon('heroicon-o-wrench-screwdriver') @@ -104,116 +106,118 @@ protected function getHeaderActions(): array ->preserveVisibility() ->requireCapability(Capabilities::TENANT_MANAGE) ->tooltip(UiTooltips::INSUFFICIENT_PERMISSION) - ->apply(), + ->apply(); + } - UiEnforcement::forAction( - Actions\Action::make('triage_all_matching') - ->label('Triage all matching') - ->icon('heroicon-o-check') - ->color('gray') - ->requiresConfirmation() - ->visible(fn (): bool => $this->getStatusFilterValue() === Finding::STATUS_NEW) - ->modalDescription(function (): string { - $count = $this->getAllMatchingCount(); + $actions[] = UiEnforcement::forAction( + Actions\Action::make('triage_all_matching') + ->label('Triage all matching') + ->icon('heroicon-o-check') + ->color('gray') + ->requiresConfirmation() + ->visible(fn (): bool => $this->getStatusFilterValue() === Finding::STATUS_NEW) + ->modalDescription(function (): string { + $count = $this->getAllMatchingCount(); - return "You are about to triage {$count} finding".($count === 1 ? '' : 's').' matching the current filters.'; - }) - ->form(function (): array { - $count = $this->getAllMatchingCount(); + return "You are about to triage {$count} finding".($count === 1 ? '' : 's').' matching the current filters.'; + }) + ->form(function (): array { + $count = $this->getAllMatchingCount(); - if ($count <= 100) { - return []; - } + if ($count <= 100) { + return []; + } - return [ - TextInput::make('confirmation') - ->label('Type TRIAGE to confirm') - ->required() - ->in(['TRIAGE']) - ->validationMessages([ - 'in' => 'Please type TRIAGE to confirm.', - ]), - ]; - }) - ->action(function (FindingWorkflowService $workflow): void { - $query = $this->buildAllMatchingQuery(); - $count = (clone $query)->count(); - - if ($count === 0) { - Notification::make() - ->title('No matching findings') - ->body('There are no new findings matching the current filters to triage.') - ->warning() - ->send(); - - return; - } - - $user = auth()->user(); - $tenant = \Filament\Facades\Filament::getTenant(); - - if (! $user instanceof User) { - abort(403); - } - - if (! $tenant instanceof Tenant) { - abort(404); - } - - $triagedCount = 0; - $skippedCount = 0; - $failedCount = 0; - - $query->orderBy('id')->chunkById(200, function ($findings) use ($workflow, $tenant, $user, &$triagedCount, &$skippedCount, &$failedCount): void { - foreach ($findings as $finding) { - if (! $finding instanceof Finding) { - $skippedCount++; - - continue; - } - - if (! in_array((string) $finding->status, [ - Finding::STATUS_NEW, - Finding::STATUS_REOPENED, - Finding::STATUS_ACKNOWLEDGED, - ], true)) { - $skippedCount++; - - continue; - } - - try { - $workflow->triage($finding, $tenant, $user); - $triagedCount++; - } catch (Throwable) { - $failedCount++; - } - } - }); - - $this->deselectAllTableRecords(); - $this->resetPage(); - - $body = "Triaged {$triagedCount} finding".($triagedCount === 1 ? '' : 's').'.'; - if ($skippedCount > 0) { - $body .= " Skipped {$skippedCount}."; - } - if ($failedCount > 0) { - $body .= " Failed {$failedCount}."; - } + return [ + TextInput::make('confirmation') + ->label('Type TRIAGE to confirm') + ->required() + ->in(['TRIAGE']) + ->validationMessages([ + 'in' => 'Please type TRIAGE to confirm.', + ]), + ]; + }) + ->action(function (FindingWorkflowService $workflow): void { + $query = $this->buildAllMatchingQuery(); + $count = (clone $query)->count(); + if ($count === 0) { Notification::make() - ->title('Bulk triage completed') - ->body($body) - ->status($failedCount > 0 ? 'warning' : 'success') + ->title('No matching findings') + ->body('There are no new findings matching the current filters to triage.') + ->warning() ->send(); - }) - ) - ->preserveVisibility() - ->requireCapability(Capabilities::TENANT_FINDINGS_TRIAGE) - ->tooltip(UiTooltips::INSUFFICIENT_PERMISSION) - ->apply(), - ]; + + return; + } + + $user = auth()->user(); + $tenant = \Filament\Facades\Filament::getTenant(); + + if (! $user instanceof User) { + abort(403); + } + + if (! $tenant instanceof Tenant) { + abort(404); + } + + $triagedCount = 0; + $skippedCount = 0; + $failedCount = 0; + + $query->orderBy('id')->chunkById(200, function ($findings) use ($workflow, $tenant, $user, &$triagedCount, &$skippedCount, &$failedCount): void { + foreach ($findings as $finding) { + if (! $finding instanceof Finding) { + $skippedCount++; + + continue; + } + + if (! in_array((string) $finding->status, [ + Finding::STATUS_NEW, + Finding::STATUS_REOPENED, + Finding::STATUS_ACKNOWLEDGED, + ], true)) { + $skippedCount++; + + continue; + } + + try { + $workflow->triage($finding, $tenant, $user); + $triagedCount++; + } catch (Throwable) { + $failedCount++; + } + } + }); + + $this->deselectAllTableRecords(); + $this->resetPage(); + + $body = "Triaged {$triagedCount} finding".($triagedCount === 1 ? '' : 's').'.'; + if ($skippedCount > 0) { + $body .= " Skipped {$skippedCount}."; + } + if ($failedCount > 0) { + $body .= " Failed {$failedCount}."; + } + + Notification::make() + ->title('Bulk triage completed') + ->body($body) + ->status($failedCount > 0 ? 'warning' : 'success') + ->send(); + }) + ) + ->preserveVisibility() + ->requireCapability(Capabilities::TENANT_FINDINGS_TRIAGE) + ->tooltip(UiTooltips::INSUFFICIENT_PERMISSION) + ->apply(); + + return $actions; } protected function buildAllMatchingQuery(): Builder diff --git a/app/Filament/System/Pages/Ops/Runbooks.php b/app/Filament/System/Pages/Ops/Runbooks.php new file mode 100644 index 0000000..e876caf --- /dev/null +++ b/app/Filament/System/Pages/Ops/Runbooks.php @@ -0,0 +1,255 @@ +scopeMode === FindingsLifecycleBackfillScope::MODE_ALL_TENANTS) { + return 'All tenants'; + } + + $tenantName = $this->selectedTenantName(); + + if ($tenantName !== null) { + return "Single tenant ({$tenantName})"; + } + + return $this->tenantId !== null ? "Single tenant (#{$this->tenantId})" : 'Single tenant'; + } + + public function selectedTenantName(): ?string + { + if ($this->tenantId === null) { + return null; + } + + return Tenant::query()->whereKey($this->tenantId)->value('name'); + } + + public static function canAccess(): bool + { + $user = auth('platform')->user(); + + if (! $user instanceof PlatformUser) { + return false; + } + + return $user->hasCapability(PlatformCapabilities::OPS_VIEW) + && $user->hasCapability(PlatformCapabilities::RUNBOOKS_VIEW); + } + + /** + * @return array + */ + protected function getHeaderActions(): array + { + return [ + Action::make('preflight') + ->label('Preflight') + ->color('gray') + ->icon('heroicon-o-magnifying-glass') + ->form($this->scopeForm()) + ->action(function (array $data, FindingsLifecycleBackfillRunbookService $runbookService): void { + $scope = FindingsLifecycleBackfillScope::fromArray([ + 'mode' => $data['scope_mode'] ?? null, + 'tenant_id' => $data['tenant_id'] ?? null, + ]); + + $this->scopeMode = $scope->mode; + $this->tenantId = $scope->tenantId; + + $this->preflight = $runbookService->preflight($scope); + + Notification::make() + ->title('Preflight complete') + ->success() + ->send(); + }), + + Action::make('run') + ->label('Run…') + ->icon('heroicon-o-play') + ->color('danger') + ->requiresConfirmation() + ->modalHeading('Run: Rebuild Findings Lifecycle') + ->modalDescription('This operation may modify customer data. Review the preflight and confirm before running.') + ->form($this->runForm()) + ->disabled(fn (): bool => ! is_array($this->preflight) || (int) ($this->preflight['affected_count'] ?? 0) <= 0) + ->action(function (array $data, FindingsLifecycleBackfillRunbookService $runbookService): void { + if (! is_array($this->preflight) || (int) ($this->preflight['affected_count'] ?? 0) <= 0) { + throw ValidationException::withMessages([ + 'preflight' => 'Run preflight first.', + ]); + } + + $scope = $this->scopeMode === FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT + ? FindingsLifecycleBackfillScope::singleTenant((int) $this->tenantId) + : FindingsLifecycleBackfillScope::allTenants(); + + $user = auth('platform')->user(); + + if (! $user instanceof PlatformUser) { + abort(403); + } + + if (! $user->hasCapability(PlatformCapabilities::RUNBOOKS_RUN) + || ! $user->hasCapability(PlatformCapabilities::RUNBOOKS_FINDINGS_LIFECYCLE_BACKFILL) + ) { + abort(403); + } + + if ($scope->isAllTenants()) { + $typedConfirmation = (string) ($data['typed_confirmation'] ?? ''); + + if ($typedConfirmation !== 'BACKFILL') { + throw ValidationException::withMessages([ + 'typed_confirmation' => 'Please type BACKFILL to confirm.', + ]); + } + } + + $reason = RunbookReason::fromNullableArray([ + 'reason_code' => $data['reason_code'] ?? null, + 'reason_text' => $data['reason_text'] ?? null, + ]); + + $run = $runbookService->start( + scope: $scope, + initiator: $user, + reason: $reason, + source: 'system_ui', + ); + + $viewUrl = SystemOperationRunLinks::view($run); + + $toast = $run->wasRecentlyCreated + ? OperationUxPresenter::queuedToast((string) $run->type)->body('The runbook will execute in the background.') + : OperationUxPresenter::alreadyQueuedToast((string) $run->type); + + $toast + ->actions([ + Action::make('view_run') + ->label('View run') + ->url($viewUrl), + ]) + ->send(); + }), + ]; + } + + /** + * @return array + */ + private function scopeForm(): array + { + return [ + Radio::make('scope_mode') + ->label('Scope') + ->options([ + FindingsLifecycleBackfillScope::MODE_ALL_TENANTS => 'All tenants', + FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT => 'Single tenant', + ]) + ->default($this->scopeMode) + ->required(), + + Select::make('tenant_id') + ->label('Tenant') + ->searchable() + ->visible(fn (callable $get): bool => $get('scope_mode') === FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT) + ->required(fn (callable $get): bool => $get('scope_mode') === FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT) + ->getSearchResultsUsing(function (string $search, AllowedTenantUniverse $universe): array { + return $universe + ->query() + ->where('name', 'like', "%{$search}%") + ->orderBy('name') + ->limit(25) + ->pluck('name', 'id') + ->all(); + }) + ->getOptionLabelUsing(function ($value, AllowedTenantUniverse $universe): ?string { + if (! is_numeric($value)) { + return null; + } + + return $universe + ->query() + ->whereKey((int) $value) + ->value('name'); + }), + ]; + } + + /** + * @return array + */ + private function runForm(): array + { + return [ + TextInput::make('typed_confirmation') + ->label('Type BACKFILL to confirm') + ->visible(fn (): bool => $this->scopeMode === FindingsLifecycleBackfillScope::MODE_ALL_TENANTS) + ->required(fn (): bool => $this->scopeMode === FindingsLifecycleBackfillScope::MODE_ALL_TENANTS) + ->in(['BACKFILL']) + ->validationMessages([ + 'in' => 'Please type BACKFILL to confirm.', + ]), + + Select::make('reason_code') + ->label('Reason code') + ->options(RunbookReason::options()) + ->required(function (BreakGlassSession $breakGlass): bool { + return $this->scopeMode === FindingsLifecycleBackfillScope::MODE_ALL_TENANTS || $breakGlass->isActive(); + }), + + Textarea::make('reason_text') + ->label('Reason') + ->rows(4) + ->maxLength(500) + ->required(function (BreakGlassSession $breakGlass): bool { + return $this->scopeMode === FindingsLifecycleBackfillScope::MODE_ALL_TENANTS || $breakGlass->isActive(); + }), + ]; + } +} diff --git a/app/Filament/System/Pages/Ops/Runs.php b/app/Filament/System/Pages/Ops/Runs.php new file mode 100644 index 0000000..5e9a1ab --- /dev/null +++ b/app/Filament/System/Pages/Ops/Runs.php @@ -0,0 +1,104 @@ +user(); + + if (! $user instanceof PlatformUser) { + return false; + } + + return $user->hasCapability(PlatformCapabilities::OPS_VIEW) + && $user->hasCapability(PlatformCapabilities::RUNBOOKS_VIEW); + } + + public function mount(): void + { + $this->mountInteractsWithTable(); + } + + public function table(Table $table): Table + { + return $table + ->defaultSort('id', 'desc') + ->query(function (): Builder { + $platformTenant = Tenant::query()->where('external_id', 'platform')->first(); + + $workspaceId = $platformTenant instanceof Tenant ? (int) $platformTenant->workspace_id : null; + + return OperationRun::query() + ->with('tenant') + ->when($workspaceId, fn (Builder $query): Builder => $query->where('workspace_id', $workspaceId)) + ->when(! $workspaceId, fn (Builder $query): Builder => $query->whereRaw('1 = 0')) + ->where('type', FindingsLifecycleBackfillRunbookService::RUNBOOK_KEY); + }) + ->columns([ + TextColumn::make('status') + ->badge() + ->formatStateUsing(BadgeRenderer::label(BadgeDomain::OperationRunStatus)) + ->color(BadgeRenderer::color(BadgeDomain::OperationRunStatus)) + ->icon(BadgeRenderer::icon(BadgeDomain::OperationRunStatus)) + ->iconColor(BadgeRenderer::iconColor(BadgeDomain::OperationRunStatus)), + TextColumn::make('scope') + ->label('Scope') + ->getStateUsing(function (OperationRun $record): string { + $scope = (string) data_get($record->context, 'runbook.scope', 'unknown'); + $tenantName = $record->tenant instanceof Tenant ? $record->tenant->name : null; + + if ($scope === 'single_tenant' && $tenantName) { + return "Single tenant ({$tenantName})"; + } + + return $scope === 'all_tenants' ? 'All tenants' : $scope; + }), + TextColumn::make('initiator_name')->label('Initiator'), + TextColumn::make('created_at')->label('Started')->since(), + TextColumn::make('outcome') + ->badge() + ->formatStateUsing(BadgeRenderer::label(BadgeDomain::OperationRunOutcome)) + ->color(BadgeRenderer::color(BadgeDomain::OperationRunOutcome)) + ->icon(BadgeRenderer::icon(BadgeDomain::OperationRunOutcome)) + ->iconColor(BadgeRenderer::iconColor(BadgeDomain::OperationRunOutcome)), + ]) + ->actions([ + Action::make('view_run') + ->label('View run') + ->url(fn (OperationRun $record): string => SystemOperationRunLinks::view($record)), + ]) + ->bulkActions([]); + } +} diff --git a/app/Filament/System/Pages/Ops/ViewRun.php b/app/Filament/System/Pages/Ops/ViewRun.php new file mode 100644 index 0000000..c84cf36 --- /dev/null +++ b/app/Filament/System/Pages/Ops/ViewRun.php @@ -0,0 +1,54 @@ +user(); + + if (! $user instanceof PlatformUser) { + return false; + } + + return $user->hasCapability(PlatformCapabilities::OPS_VIEW) + && $user->hasCapability(PlatformCapabilities::RUNBOOKS_VIEW); + } + + public function mount(OperationRun $run): void + { + $platformTenant = Tenant::query()->where('external_id', 'platform')->first(); + + $workspaceId = $platformTenant instanceof Tenant ? (int) $platformTenant->workspace_id : null; + + $run->load('tenant'); + + if ($workspaceId === null || (int) $run->workspace_id !== $workspaceId) { + abort(404); + } + + if ((string) $run->type !== FindingsLifecycleBackfillRunbookService::RUNBOOK_KEY) { + abort(404); + } + + $this->run = $run; + } +} diff --git a/app/Jobs/BackfillFindingLifecycleJob.php b/app/Jobs/BackfillFindingLifecycleJob.php index a9d629d..46be503 100644 --- a/app/Jobs/BackfillFindingLifecycleJob.php +++ b/app/Jobs/BackfillFindingLifecycleJob.php @@ -9,6 +9,7 @@ use App\Models\User; use App\Services\Findings\FindingSlaPolicy; use App\Services\OperationRunService; +use App\Services\Runbooks\FindingsLifecycleBackfillRunbookService; use App\Support\OperationRunOutcome; use App\Support\OperationRunStatus; use App\Support\OpsUx\RunFailureSanitizer; @@ -33,8 +34,11 @@ public function __construct( public readonly ?int $initiatorUserId = null, ) {} - public function handle(OperationRunService $operationRuns, FindingSlaPolicy $slaPolicy): void - { + public function handle( + OperationRunService $operationRuns, + FindingSlaPolicy $slaPolicy, + FindingsLifecycleBackfillRunbookService $runbookService, + ): void { $tenant = Tenant::query()->find($this->tenantId); if (! $tenant instanceof Tenant) { @@ -76,6 +80,8 @@ public function handle(OperationRunService $operationRuns, FindingSlaPolicy $sla ); } + $runbookService->maybeFinalize($operationRun); + return; } @@ -154,6 +160,8 @@ public function handle(OperationRunService $operationRuns, FindingSlaPolicy $sla status: OperationRunStatus::Completed->value, outcome: OperationRunOutcome::Succeeded->value, ); + + $runbookService->maybeFinalize($operationRun); } catch (Throwable $e) { $message = RunFailureSanitizer::sanitizeMessage($e->getMessage()); $reasonCode = RunFailureSanitizer::normalizeReasonCode($e->getMessage()); @@ -169,6 +177,8 @@ public function handle(OperationRunService $operationRuns, FindingSlaPolicy $sla ]], ); + $runbookService->maybeFinalize($operationRun); + throw $e; } finally { $lock->release(); diff --git a/app/Jobs/BackfillFindingLifecycleTenantIntoWorkspaceRunJob.php b/app/Jobs/BackfillFindingLifecycleTenantIntoWorkspaceRunJob.php new file mode 100644 index 0000000..a4c06c3 --- /dev/null +++ b/app/Jobs/BackfillFindingLifecycleTenantIntoWorkspaceRunJob.php @@ -0,0 +1,375 @@ +find($this->tenantId); + + if (! $tenant instanceof Tenant) { + return; + } + + if ((int) $tenant->workspace_id !== $this->workspaceId) { + return; + } + + $run = OperationRun::query()->find($this->operationRunId); + + if (! $run instanceof OperationRun) { + return; + } + + if ((int) $run->workspace_id !== $this->workspaceId) { + return; + } + + if ($run->tenant_id !== null) { + return; + } + + if ($run->status === 'queued') { + $operationRunService->updateRun($run, status: 'running'); + } + + $lock = Cache::lock(sprintf('tenantpilot:findings:lifecycle_backfill:tenant:%d', $this->tenantId), 900); + + if (! $lock->get()) { + $operationRunService->appendFailures($run, [ + [ + 'code' => 'findings.lifecycle.backfill.lock_busy', + 'message' => sprintf('Tenant %d is already running a findings lifecycle backfill.', $this->tenantId), + ], + ]); + + $operationRunService->incrementSummaryCounts($run, [ + 'failed' => 1, + 'processed' => 1, + ]); + + $operationRunService->maybeCompleteBulkRun($run); + $runbookService->maybeFinalize($run); + + return; + } + + try { + $backfillStartedAt = $run->started_at !== null + ? CarbonImmutable::instance($run->started_at) + : CarbonImmutable::now('UTC'); + + Finding::query() + ->where('tenant_id', $tenant->getKey()) + ->orderBy('id') + ->chunkById(200, function (Collection $findings) use ($tenant, $slaPolicy, $operationRunService, $run, $backfillStartedAt): void { + $updated = 0; + $skipped = 0; + + foreach ($findings as $finding) { + if (! $finding instanceof Finding) { + continue; + } + + $originalAttributes = $finding->getAttributes(); + + $this->backfillLifecycleFields($finding, $backfillStartedAt); + $this->backfillLegacyAcknowledgedStatus($finding); + $this->backfillSlaFields($finding, $tenant, $slaPolicy, $backfillStartedAt); + $this->backfillDriftRecurrenceKey($finding); + + if ($finding->isDirty()) { + $finding->save(); + $updated++; + } else { + $finding->setRawAttributes($originalAttributes, sync: true); + $skipped++; + } + } + + if ($updated > 0 || $skipped > 0) { + $operationRunService->incrementSummaryCounts($run, [ + 'updated' => $updated, + 'skipped' => $skipped, + ]); + } + }); + + $consolidatedDuplicates = $this->consolidateDriftDuplicates($tenant, $backfillStartedAt); + + if ($consolidatedDuplicates > 0) { + $operationRunService->incrementSummaryCounts($run, [ + 'updated' => $consolidatedDuplicates, + ]); + } + + $operationRunService->incrementSummaryCounts($run, [ + 'processed' => 1, + ]); + + $operationRunService->maybeCompleteBulkRun($run); + $runbookService->maybeFinalize($run); + } catch (Throwable $e) { + $message = RunFailureSanitizer::sanitizeMessage($e->getMessage()); + $reasonCode = RunFailureSanitizer::normalizeReasonCode($e->getMessage()); + + $operationRunService->appendFailures($run, [[ + 'code' => 'findings.lifecycle.backfill.failed', + 'reason_code' => $reasonCode, + 'message' => $message !== '' ? $message : sprintf('Tenant %d findings lifecycle backfill failed.', $this->tenantId), + ]]); + + $operationRunService->incrementSummaryCounts($run, [ + 'failed' => 1, + 'processed' => 1, + ]); + + $operationRunService->maybeCompleteBulkRun($run); + $runbookService->maybeFinalize($run); + + throw $e; + } finally { + $lock->release(); + } + } + + private function backfillLifecycleFields(Finding $finding, CarbonImmutable $backfillStartedAt): void + { + $createdAt = $finding->created_at !== null ? CarbonImmutable::instance($finding->created_at) : $backfillStartedAt; + + if ($finding->first_seen_at === null) { + $finding->first_seen_at = $createdAt; + } + + if ($finding->last_seen_at === null) { + $finding->last_seen_at = $createdAt; + } + + if ($finding->last_seen_at !== null && $finding->first_seen_at !== null) { + $lastSeen = CarbonImmutable::instance($finding->last_seen_at); + $firstSeen = CarbonImmutable::instance($finding->first_seen_at); + + if ($lastSeen->lessThan($firstSeen)) { + $finding->last_seen_at = $firstSeen; + } + } + + $timesSeen = is_numeric($finding->times_seen) ? (int) $finding->times_seen : 0; + + if ($timesSeen < 1) { + $finding->times_seen = 1; + } + } + + private function backfillLegacyAcknowledgedStatus(Finding $finding): void + { + if ($finding->status !== Finding::STATUS_ACKNOWLEDGED) { + return; + } + + $finding->status = Finding::STATUS_TRIAGED; + + if ($finding->triaged_at === null) { + if ($finding->acknowledged_at !== null) { + $finding->triaged_at = CarbonImmutable::instance($finding->acknowledged_at); + } elseif ($finding->created_at !== null) { + $finding->triaged_at = CarbonImmutable::instance($finding->created_at); + } + } + } + + private function backfillSlaFields( + Finding $finding, + Tenant $tenant, + FindingSlaPolicy $slaPolicy, + CarbonImmutable $backfillStartedAt, + ): void { + if (! Finding::isOpenStatus((string) $finding->status)) { + return; + } + + if ($finding->sla_days === null) { + $finding->sla_days = $slaPolicy->daysForSeverity((string) $finding->severity, $tenant); + } + + if ($finding->due_at === null) { + $finding->due_at = $slaPolicy->dueAtForSeverity((string) $finding->severity, $tenant, $backfillStartedAt); + } + } + + private function backfillDriftRecurrenceKey(Finding $finding): void + { + if ($finding->finding_type !== Finding::FINDING_TYPE_DRIFT) { + return; + } + + if ($finding->recurrence_key !== null && trim((string) $finding->recurrence_key) !== '') { + return; + } + + $tenantId = (int) ($finding->tenant_id ?? 0); + $scopeKey = (string) ($finding->scope_key ?? ''); + $subjectType = (string) ($finding->subject_type ?? ''); + $subjectExternalId = (string) ($finding->subject_external_id ?? ''); + + if ($tenantId <= 0 || $scopeKey === '' || $subjectType === '' || $subjectExternalId === '') { + return; + } + + $evidence = is_array($finding->evidence_jsonb) ? $finding->evidence_jsonb : []; + $kind = Arr::get($evidence, 'summary.kind'); + $changeType = Arr::get($evidence, 'change_type'); + + $kind = is_string($kind) ? $kind : ''; + $changeType = is_string($changeType) ? $changeType : ''; + + if ($kind === '') { + return; + } + + $dimension = $this->recurrenceDimension($kind, $changeType); + + $finding->recurrence_key = hash('sha256', sprintf( + 'drift:%d:%s:%s:%s:%s', + $tenantId, + $scopeKey, + $subjectType, + $subjectExternalId, + $dimension, + )); + } + + private function recurrenceDimension(string $kind, string $changeType): string + { + $kind = strtolower(trim($kind)); + $changeType = strtolower(trim($changeType)); + + return match ($kind) { + 'policy_snapshot', 'baseline_compare' => sprintf('%s:%s', $kind, $changeType !== '' ? $changeType : 'modified'), + default => $kind, + }; + } + + private function consolidateDriftDuplicates(Tenant $tenant, CarbonImmutable $backfillStartedAt): int + { + $duplicateKeys = Finding::query() + ->where('tenant_id', $tenant->getKey()) + ->where('finding_type', Finding::FINDING_TYPE_DRIFT) + ->whereNotNull('recurrence_key') + ->select(['recurrence_key']) + ->groupBy('recurrence_key') + ->havingRaw('COUNT(*) > 1') + ->pluck('recurrence_key') + ->filter(static fn (mixed $value): bool => is_string($value) && trim($value) !== '') + ->values(); + + if ($duplicateKeys->isEmpty()) { + return 0; + } + + $consolidated = 0; + + foreach ($duplicateKeys as $recurrenceKey) { + if (! is_string($recurrenceKey) || $recurrenceKey === '') { + continue; + } + + $findings = Finding::query() + ->where('tenant_id', $tenant->getKey()) + ->where('finding_type', Finding::FINDING_TYPE_DRIFT) + ->where('recurrence_key', $recurrenceKey) + ->orderBy('id') + ->get(); + + $canonical = $this->chooseCanonicalDriftFinding($findings, $recurrenceKey); + + foreach ($findings as $finding) { + if (! $finding instanceof Finding) { + continue; + } + + if ($canonical instanceof Finding && (int) $finding->getKey() === (int) $canonical->getKey()) { + continue; + } + + $finding->forceFill([ + 'status' => Finding::STATUS_RESOLVED, + 'resolved_at' => $backfillStartedAt, + 'resolved_reason' => 'consolidated_duplicate', + 'recurrence_key' => null, + ])->save(); + + $consolidated++; + } + } + + return $consolidated; + } + + /** + * @param Collection $findings + */ + private function chooseCanonicalDriftFinding(Collection $findings, string $recurrenceKey): ?Finding + { + if ($findings->isEmpty()) { + return null; + } + + $openCandidates = $findings->filter(static fn (Finding $finding): bool => Finding::isOpenStatus((string) $finding->status)); + + $candidates = $openCandidates->isNotEmpty() ? $openCandidates : $findings; + + $alreadyCanonical = $candidates->first(static fn (Finding $finding): bool => (string) $finding->fingerprint === $recurrenceKey); + + if ($alreadyCanonical instanceof Finding) { + return $alreadyCanonical; + } + + /** @var Finding $sorted */ + $sorted = $candidates + ->sortByDesc(function (Finding $finding): array { + $lastSeen = $finding->last_seen_at !== null ? CarbonImmutable::instance($finding->last_seen_at)->getTimestamp() : 0; + $createdAt = $finding->created_at !== null ? CarbonImmutable::instance($finding->created_at)->getTimestamp() : 0; + + return [ + max($lastSeen, $createdAt), + (int) $finding->getKey(), + ]; + }) + ->first(); + + return $sorted; + } +} diff --git a/app/Jobs/BackfillFindingLifecycleWorkspaceJob.php b/app/Jobs/BackfillFindingLifecycleWorkspaceJob.php new file mode 100644 index 0000000..f7e93a9 --- /dev/null +++ b/app/Jobs/BackfillFindingLifecycleWorkspaceJob.php @@ -0,0 +1,95 @@ +find($this->operationRunId); + + if (! $run instanceof OperationRun) { + return; + } + + if ((int) $run->workspace_id !== $this->workspaceId) { + return; + } + + if ($run->tenant_id !== null) { + return; + } + + $tenantIds = $allowedTenantUniverse + ->query() + ->where('workspace_id', $this->workspaceId) + ->orderBy('id') + ->pluck('id') + ->map(static fn (mixed $id): int => (int) $id) + ->all(); + + $tenantCount = count($tenantIds); + + $operationRunService->updateRun( + $run, + status: OperationRunStatus::Running->value, + outcome: OperationRunOutcome::Pending->value, + summaryCounts: [ + 'tenants' => $tenantCount, + 'total' => $tenantCount, + 'processed' => 0, + 'updated' => 0, + 'skipped' => 0, + 'failed' => 0, + ], + ); + + if ($tenantCount === 0) { + $operationRunService->updateRun( + $run, + status: OperationRunStatus::Completed->value, + outcome: OperationRunOutcome::Succeeded->value, + ); + + $runbookService->maybeFinalize($run); + + return; + } + + foreach ($tenantIds as $tenantId) { + if ($tenantId <= 0) { + continue; + } + + BackfillFindingLifecycleTenantIntoWorkspaceRunJob::dispatch( + operationRunId: (int) $run->getKey(), + workspaceId: $this->workspaceId, + tenantId: $tenantId, + ); + } + } +} diff --git a/app/Notifications/OperationRunCompleted.php b/app/Notifications/OperationRunCompleted.php index 807abc9..ee09b56 100644 --- a/app/Notifications/OperationRunCompleted.php +++ b/app/Notifications/OperationRunCompleted.php @@ -3,8 +3,10 @@ namespace App\Notifications; use App\Models\OperationRun; +use App\Models\PlatformUser; use App\Models\Tenant; use App\Support\OpsUx\OperationUxPresenter; +use App\Support\System\SystemOperationRunLinks; use Illuminate\Bus\Queueable; use Illuminate\Notifications\Notification; @@ -25,9 +27,19 @@ public function toDatabase(object $notifiable): array { $tenant = $this->run->tenant; - return OperationUxPresenter::terminalDatabaseNotification( + $notification = OperationUxPresenter::terminalDatabaseNotification( run: $this->run, tenant: $tenant instanceof Tenant ? $tenant : null, - )->getDatabaseMessage(); + ); + + if ($notifiable instanceof PlatformUser) { + $notification->actions([ + \Filament\Actions\Action::make('view_run') + ->label('View run') + ->url(SystemOperationRunLinks::view($this->run)), + ]); + } + + return $notification->getDatabaseMessage(); } } diff --git a/app/Providers/Filament/SystemPanelProvider.php b/app/Providers/Filament/SystemPanelProvider.php index 27a4997..c68c273 100644 --- a/app/Providers/Filament/SystemPanelProvider.php +++ b/app/Providers/Filament/SystemPanelProvider.php @@ -2,9 +2,9 @@ namespace App\Providers\Filament; -use App\Http\Middleware\UseSystemSessionCookie; use App\Filament\System\Pages\Auth\Login; use App\Filament\System\Pages\Dashboard; +use App\Http\Middleware\UseSystemSessionCookie; use App\Support\Auth\PlatformCapabilities; use Filament\Http\Middleware\Authenticate; use Filament\Http\Middleware\AuthenticateSession; @@ -33,6 +33,7 @@ public function panel(Panel $panel): Panel ->colors([ 'primary' => Color::Blue, ]) + ->databaseNotifications() ->renderHook( PanelsRenderHook::BODY_START, fn () => view('filament.system.components.break-glass-banner')->render(), diff --git a/app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php b/app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php new file mode 100644 index 0000000..9874237 --- /dev/null +++ b/app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php @@ -0,0 +1,609 @@ +computePreflight($scope); + + $this->auditSafely( + action: 'platform.ops.runbooks.preflight', + scope: $scope, + operationRunId: null, + context: [ + 'preflight' => $result, + ], + ); + + return $result; + } + + public function start( + FindingsLifecycleBackfillScope $scope, + ?PlatformUser $initiator, + ?RunbookReason $reason, + string $source, + ): OperationRun { + $source = trim($source); + + if ($source === '') { + throw ValidationException::withMessages([ + 'source' => 'A run source is required.', + ]); + } + + $isBreakGlassActive = $this->breakGlassSession->isActive(); + + if ($scope->isAllTenants() || $isBreakGlassActive) { + if (! $reason instanceof RunbookReason) { + throw ValidationException::withMessages([ + 'reason' => 'A reason is required for this run.', + ]); + } + } + + $preflight = $this->computePreflight($scope); + + if (($preflight['affected_count'] ?? 0) <= 0) { + throw ValidationException::withMessages([ + 'preflight.affected_count' => 'Nothing to do for this scope.', + ]); + } + + $platformTenant = $this->platformTenant(); + $workspace = $platformTenant->workspace; + + if (! $workspace instanceof Workspace) { + throw new \RuntimeException('Platform tenant is missing its workspace.'); + } + + if ($scope->isAllTenants()) { + $lockKey = sprintf('tenantpilot:runbooks:%s:workspace:%d', self::RUNBOOK_KEY, (int) $workspace->getKey()); + $lock = Cache::lock($lockKey, 900); + + if (! $lock->get()) { + throw ValidationException::withMessages([ + 'scope' => 'Another run is already in progress for this scope.', + ]); + } + + try { + return $this->startAllTenants( + workspace: $workspace, + initiator: $initiator, + reason: $reason, + preflight: $preflight, + source: $source, + isBreakGlassActive: $isBreakGlassActive, + ); + } finally { + $lock->release(); + } + } + + return $this->startSingleTenant( + tenantId: (int) $scope->tenantId, + initiator: $initiator, + reason: $reason, + preflight: $preflight, + source: $source, + isBreakGlassActive: $isBreakGlassActive, + ); + } + + public function maybeFinalize(OperationRun $run): void + { + $run->refresh(); + + if ($run->status !== OperationRunStatus::Completed->value) { + return; + } + + $context = is_array($run->context) ? $run->context : []; + + if ((string) data_get($context, 'runbook.key') !== self::RUNBOOK_KEY) { + return; + } + + $lockKey = sprintf('tenantpilot:runbooks:%s:finalize:%d', self::RUNBOOK_KEY, (int) $run->getKey()); + $lock = Cache::lock($lockKey, 86400); + + if (! $lock->get()) { + return; + } + + try { + $this->auditSafely( + action: $run->outcome === OperationRunOutcome::Failed->value + ? 'platform.ops.runbooks.failed' + : 'platform.ops.runbooks.completed', + scope: $this->scopeFromRunContext($context), + operationRunId: (int) $run->getKey(), + context: [ + 'status' => (string) $run->status, + 'outcome' => (string) $run->outcome, + 'is_break_glass' => (bool) data_get($context, 'platform_initiator.is_break_glass', false), + 'reason_code' => data_get($context, 'reason.reason_code'), + 'reason_text' => data_get($context, 'reason.reason_text'), + ], + ); + + $this->notifyInitiatorSafely($run); + + if ($run->outcome === OperationRunOutcome::Failed->value) { + $this->dispatchFailureAlertSafely($run); + } + } finally { + $lock->release(); + } + } + + /** + * @return array{affected_count: int, total_count: int, estimated_tenants?: int|null} + */ + private function computePreflight(FindingsLifecycleBackfillScope $scope): array + { + if ($scope->isSingleTenant()) { + $tenant = Tenant::query()->whereKey((int) $scope->tenantId)->firstOrFail(); + $this->allowedTenantUniverse->ensureAllowed($tenant); + + return $this->computeTenantPreflight($tenant); + } + + $platformTenant = $this->platformTenant(); + $workspaceId = (int) ($platformTenant->workspace_id ?? 0); + + $tenants = $this->allowedTenantUniverse + ->query() + ->where('workspace_id', $workspaceId) + ->orderBy('id') + ->get(); + + $affected = 0; + $total = 0; + + foreach ($tenants as $tenant) { + if (! $tenant instanceof Tenant) { + continue; + } + + $counts = $this->computeTenantPreflight($tenant); + + $affected += (int) ($counts['affected_count'] ?? 0); + $total += (int) ($counts['total_count'] ?? 0); + } + + return [ + 'affected_count' => $affected, + 'total_count' => $total, + 'estimated_tenants' => $tenants->count(), + ]; + } + + /** + * @return array{affected_count: int, total_count: int} + */ + private function computeTenantPreflight(Tenant $tenant): array + { + $query = Finding::query()->where('tenant_id', (int) $tenant->getKey()); + + $total = (int) (clone $query)->count(); + + $affected = 0; + + (clone $query) + ->orderBy('id') + ->chunkById(500, function ($findings) use (&$affected): void { + foreach ($findings as $finding) { + if (! $finding instanceof Finding) { + continue; + } + + if ($this->findingNeedsBackfill($finding)) { + $affected++; + } + } + }); + + $affected += $this->countDriftDuplicateConsolidations($tenant); + + return [ + 'affected_count' => $affected, + 'total_count' => $total, + ]; + } + + private function findingNeedsBackfill(Finding $finding): bool + { + if ($finding->first_seen_at === null || $finding->last_seen_at === null) { + return true; + } + + if ($finding->last_seen_at !== null && $finding->first_seen_at !== null) { + if ($finding->last_seen_at->lt($finding->first_seen_at)) { + return true; + } + } + + $timesSeen = is_numeric($finding->times_seen) ? (int) $finding->times_seen : 0; + + if ($timesSeen < 1) { + return true; + } + + if ($finding->status === Finding::STATUS_ACKNOWLEDGED) { + return true; + } + + if (Finding::isOpenStatus((string) $finding->status)) { + if ($finding->sla_days === null || $finding->due_at === null) { + return true; + } + } + + if ($finding->finding_type === Finding::FINDING_TYPE_DRIFT) { + $recurrenceKey = $finding->recurrence_key !== null ? trim((string) $finding->recurrence_key) : ''; + + if ($recurrenceKey === '') { + $scopeKey = trim((string) ($finding->scope_key ?? '')); + $subjectType = trim((string) ($finding->subject_type ?? '')); + $subjectExternalId = trim((string) ($finding->subject_external_id ?? '')); + + if ($scopeKey !== '' && $subjectType !== '' && $subjectExternalId !== '') { + $evidence = is_array($finding->evidence_jsonb) ? $finding->evidence_jsonb : []; + $kind = data_get($evidence, 'summary.kind'); + + if (is_string($kind) && trim($kind) !== '') { + return true; + } + } + } + } + + return false; + } + + private function countDriftDuplicateConsolidations(Tenant $tenant): int + { + $rows = Finding::query() + ->where('tenant_id', (int) $tenant->getKey()) + ->where('finding_type', Finding::FINDING_TYPE_DRIFT) + ->whereNotNull('recurrence_key') + ->select(['recurrence_key', DB::raw('COUNT(*) as count')]) + ->groupBy('recurrence_key') + ->havingRaw('COUNT(*) > 1') + ->get(); + + $duplicates = 0; + + foreach ($rows as $row) { + $count = is_numeric($row->count ?? null) ? (int) $row->count : 0; + + if ($count > 1) { + $duplicates += ($count - 1); + } + } + + return $duplicates; + } + + private function startAllTenants( + Workspace $workspace, + ?PlatformUser $initiator, + ?RunbookReason $reason, + array $preflight, + string $source, + bool $isBreakGlassActive, + ): OperationRun { + $run = $this->operationRunService->ensureWorkspaceRunWithIdentity( + workspace: $workspace, + type: self::RUNBOOK_KEY, + identityInputs: [ + 'runbook' => self::RUNBOOK_KEY, + 'scope' => FindingsLifecycleBackfillScope::MODE_ALL_TENANTS, + ], + context: $this->buildRunContext( + workspaceId: (int) $workspace->getKey(), + scope: FindingsLifecycleBackfillScope::allTenants(), + initiator: $initiator, + reason: $reason, + preflight: $preflight, + source: $source, + isBreakGlassActive: $isBreakGlassActive, + ), + initiator: null, + ); + + if ($initiator instanceof PlatformUser && $run->wasRecentlyCreated) { + $run->update(['initiator_name' => $initiator->name ?: $initiator->email]); + $run->refresh(); + } + + $this->auditSafely( + action: 'platform.ops.runbooks.start', + scope: FindingsLifecycleBackfillScope::allTenants(), + operationRunId: (int) $run->getKey(), + context: [ + 'preflight' => $preflight, + 'is_break_glass' => $isBreakGlassActive, + ] + ($reason instanceof RunbookReason ? $reason->toArray() : []), + ); + + if (! $run->wasRecentlyCreated) { + return $run; + } + + $this->operationRunService->dispatchOrFail($run, function () use ($run, $workspace): void { + BackfillFindingLifecycleWorkspaceJob::dispatch( + operationRunId: (int) $run->getKey(), + workspaceId: (int) $workspace->getKey(), + ); + }); + + return $run; + } + + private function startSingleTenant( + int $tenantId, + ?PlatformUser $initiator, + ?RunbookReason $reason, + array $preflight, + string $source, + bool $isBreakGlassActive, + ): OperationRun { + $tenant = Tenant::query()->whereKey($tenantId)->firstOrFail(); + $this->allowedTenantUniverse->ensureAllowed($tenant); + + $run = $this->operationRunService->ensureRunWithIdentity( + tenant: $tenant, + type: self::RUNBOOK_KEY, + identityInputs: [ + 'tenant_id' => (int) $tenant->getKey(), + 'trigger' => 'backfill', + ], + context: $this->buildRunContext( + workspaceId: (int) $tenant->workspace_id, + scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), + initiator: $initiator, + reason: $reason, + preflight: $preflight, + source: $source, + isBreakGlassActive: $isBreakGlassActive, + ), + initiator: null, + ); + + if ($initiator instanceof PlatformUser && $run->wasRecentlyCreated) { + $run->update(['initiator_name' => $initiator->name ?: $initiator->email]); + $run->refresh(); + } + + $this->auditSafely( + action: 'platform.ops.runbooks.start', + scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), + operationRunId: (int) $run->getKey(), + context: [ + 'preflight' => $preflight, + 'is_break_glass' => $isBreakGlassActive, + ] + ($reason instanceof RunbookReason ? $reason->toArray() : []), + ); + + if (! $run->wasRecentlyCreated) { + return $run; + } + + $this->operationRunService->dispatchOrFail($run, function () use ($tenant): void { + BackfillFindingLifecycleJob::dispatch( + tenantId: (int) $tenant->getKey(), + workspaceId: (int) $tenant->workspace_id, + initiatorUserId: null, + ); + }); + + return $run; + } + + private function platformTenant(): Tenant + { + $tenant = Tenant::query()->where('external_id', 'platform')->first(); + + if (! $tenant instanceof Tenant) { + throw new \RuntimeException('Platform tenant is missing.'); + } + + return $tenant; + } + + /** + * @return array + */ + private function buildRunContext( + int $workspaceId, + FindingsLifecycleBackfillScope $scope, + ?PlatformUser $initiator, + ?RunbookReason $reason, + array $preflight, + string $source, + bool $isBreakGlassActive, + ): array { + $context = [ + 'workspace_id' => $workspaceId, + 'runbook' => [ + 'key' => self::RUNBOOK_KEY, + 'scope' => $scope->mode, + 'target_tenant_id' => $scope->tenantId, + 'source' => $source, + ], + 'preflight' => [ + 'affected_count' => (int) ($preflight['affected_count'] ?? 0), + 'total_count' => (int) ($preflight['total_count'] ?? 0), + 'estimated_tenants' => $preflight['estimated_tenants'] ?? null, + ], + ]; + + if ($reason instanceof RunbookReason) { + $context['reason'] = $reason->toArray(); + } + + if ($initiator instanceof PlatformUser) { + $context['platform_initiator'] = [ + 'platform_user_id' => (int) $initiator->getKey(), + 'email' => (string) $initiator->email, + 'name' => (string) $initiator->name, + 'is_break_glass' => $isBreakGlassActive, + ]; + } + + return $context; + } + + private function scopeFromRunContext(array $context): FindingsLifecycleBackfillScope + { + $scope = data_get($context, 'runbook.scope'); + $tenantId = data_get($context, 'runbook.target_tenant_id'); + + if ($scope === FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT && is_numeric($tenantId)) { + return FindingsLifecycleBackfillScope::singleTenant((int) $tenantId); + } + + return FindingsLifecycleBackfillScope::allTenants(); + } + + /** + * @param array $context + */ + private function auditSafely( + string $action, + FindingsLifecycleBackfillScope $scope, + ?int $operationRunId, + array $context = [], + ): void { + try { + $platformTenant = $this->platformTenant(); + + $actor = auth('platform')->user(); + + $actorId = null; + $actorEmail = null; + $actorName = null; + + if ($actor instanceof PlatformUser) { + $actorId = (int) $actor->getKey(); + $actorEmail = (string) $actor->email; + $actorName = (string) $actor->name; + } + + $metadata = [ + 'runbook_key' => self::RUNBOOK_KEY, + 'scope' => $scope->mode, + 'target_tenant_id' => $scope->tenantId, + 'operation_run_id' => $operationRunId, + 'ip' => request()->ip(), + 'user_agent' => request()->userAgent(), + ]; + + $this->auditLogger->log( + tenant: $platformTenant, + action: $action, + context: [ + 'metadata' => array_filter($metadata, static fn (mixed $value): bool => $value !== null), + ] + $context, + actorId: $actorId, + actorEmail: $actorEmail, + actorName: $actorName, + status: 'success', + resourceType: 'operation_run', + resourceId: $operationRunId !== null ? (string) $operationRunId : null, + ); + } catch (Throwable) { + // Audit is fail-safe (must not crash runbooks). + } + } + + private function notifyInitiatorSafely(OperationRun $run): void + { + try { + $platformUserId = data_get($run->context, 'platform_initiator.platform_user_id'); + + if (! is_numeric($platformUserId)) { + return; + } + + $platformUser = PlatformUser::query()->whereKey((int) $platformUserId)->first(); + + if (! $platformUser instanceof PlatformUser) { + return; + } + + $platformUser->notify(new OperationRunCompleted($run)); + } catch (Throwable) { + // Notifications must not crash the runbook. + } + } + + private function dispatchFailureAlertSafely(OperationRun $run): void + { + try { + $platformTenant = $this->platformTenant(); + $workspace = $platformTenant->workspace; + + if (! $workspace instanceof Workspace) { + return; + } + + $this->alertDispatchService->dispatchEvent($workspace, [ + 'tenant_id' => (int) $platformTenant->getKey(), + 'event_type' => 'operations.run.failed', + 'severity' => 'high', + 'title' => 'Operation failed: Findings lifecycle backfill', + 'body' => 'A findings lifecycle backfill run failed.', + 'metadata' => [ + 'operation_run_id' => (int) $run->getKey(), + 'operation_type' => (string) $run->type, + 'scope' => (string) data_get($run->context, 'runbook.scope', ''), + 'view_run_url' => SystemOperationRunLinks::view($run), + ], + ]); + } catch (Throwable) { + // Alerts must not crash the runbook. + } + } +} diff --git a/app/Services/Runbooks/FindingsLifecycleBackfillScope.php b/app/Services/Runbooks/FindingsLifecycleBackfillScope.php new file mode 100644 index 0000000..3c913f7 --- /dev/null +++ b/app/Services/Runbooks/FindingsLifecycleBackfillScope.php @@ -0,0 +1,81 @@ + 'Select a valid tenant.', + ]); + } + + return new self( + mode: self::MODE_SINGLE_TENANT, + tenantId: $tenantId, + ); + } + + /** + * @param array $data + */ + public static function fromArray(array $data): self + { + $mode = trim((string) ($data['mode'] ?? '')); + + if ($mode === '' || $mode === self::MODE_ALL_TENANTS) { + return self::allTenants(); + } + + if ($mode !== self::MODE_SINGLE_TENANT) { + throw ValidationException::withMessages([ + 'scope.mode' => 'Select a valid scope mode.', + ]); + } + + $tenantId = $data['tenant_id'] ?? null; + + if (! is_numeric($tenantId)) { + throw ValidationException::withMessages([ + 'scope.tenant_id' => 'Select a tenant.', + ]); + } + + return self::singleTenant((int) $tenantId); + } + + public function isAllTenants(): bool + { + return $this->mode === self::MODE_ALL_TENANTS; + } + + public function isSingleTenant(): bool + { + return $this->mode === self::MODE_SINGLE_TENANT; + } +} diff --git a/app/Services/Runbooks/RunbookReason.php b/app/Services/Runbooks/RunbookReason.php new file mode 100644 index 0000000..d4ca4cc --- /dev/null +++ b/app/Services/Runbooks/RunbookReason.php @@ -0,0 +1,103 @@ +reasonCode); + $reasonText = trim($this->reasonText); + + if (! in_array($reasonCode, self::codes(), true)) { + throw ValidationException::withMessages([ + 'reason.reason_code' => 'Select a valid reason code.', + ]); + } + + if ($reasonText === '') { + throw ValidationException::withMessages([ + 'reason.reason_text' => 'Provide a reason.', + ]); + } + + if (mb_strlen($reasonText) > 500) { + throw ValidationException::withMessages([ + 'reason.reason_text' => 'Reason must be 500 characters or fewer.', + ]); + } + } + + /** + * @param array|null $data + */ + public static function fromNullableArray(?array $data): ?self + { + if (! is_array($data)) { + return null; + } + + $reasonCode = trim((string) ($data['reason_code'] ?? '')); + $reasonText = trim((string) ($data['reason_text'] ?? '')); + + if ($reasonCode === '' && $reasonText === '') { + return null; + } + + return new self( + reasonCode: $reasonCode, + reasonText: $reasonText, + ); + } + + /** + * @return array + */ + public static function codes(): array + { + return [ + self::CODE_DATA_REPAIR, + self::CODE_INCIDENT, + self::CODE_SUPPORT, + self::CODE_SECURITY, + ]; + } + + /** + * @return array + */ + public static function options(): array + { + return [ + self::CODE_DATA_REPAIR => 'Data repair', + self::CODE_INCIDENT => 'Incident', + self::CODE_SUPPORT => 'Support', + self::CODE_SECURITY => 'Security', + ]; + } + + /** + * @return array{reason_code: string, reason_text: string} + */ + public function toArray(): array + { + return [ + 'reason_code' => trim($this->reasonCode), + 'reason_text' => trim($this->reasonText), + ]; + } +} diff --git a/app/Support/System/SystemOperationRunLinks.php b/app/Support/System/SystemOperationRunLinks.php new file mode 100644 index 0000000..905a238 --- /dev/null +++ b/app/Support/System/SystemOperationRunLinks.php @@ -0,0 +1,24 @@ +getKey() : (int) $run; + + return ViewRun::getUrl(['run' => $runId], panel: 'system'); + } +} diff --git a/config/tenantpilot.php b/config/tenantpilot.php index 02599b8..deaad60 100644 --- a/config/tenantpilot.php +++ b/config/tenantpilot.php @@ -6,6 +6,8 @@ 'ttl_minutes' => (int) env('BREAK_GLASS_TTL_MINUTES', 15), ], + 'allow_admin_maintenance_actions' => (bool) env('ALLOW_ADMIN_MAINTENANCE_ACTIONS', false), + 'supported_policy_types' => [ [ 'type' => 'deviceConfiguration', diff --git a/resources/views/filament/system/pages/ops/runbooks.blade.php b/resources/views/filament/system/pages/ops/runbooks.blade.php new file mode 100644 index 0000000..747738a --- /dev/null +++ b/resources/views/filament/system/pages/ops/runbooks.blade.php @@ -0,0 +1,59 @@ + +
+
+
Operator warning
+
+ Runbooks can modify customer data across tenants. Always run preflight first, and ensure you have the correct scope selected. +
+
+ +
+
+
+
+ Rebuild Findings Lifecycle +
+
+ Backfills legacy findings lifecycle fields, SLA due dates, and consolidates drift duplicate findings. +
+
+ +
+ Scope: {{ $this->scopeLabel() }} +
+
+ +
+ @if (is_array($this->preflight)) +
+
+
Affected
+
{{ (int) ($this->preflight['affected_count'] ?? 0) }}
+
+ +
+
Total scanned
+
{{ (int) ($this->preflight['total_count'] ?? 0) }}
+
+ +
+
Estimated tenants
+
{{ is_numeric($this->preflight['estimated_tenants'] ?? null) ? (int) $this->preflight['estimated_tenants'] : '—' }}
+
+
+ + @if ((int) ($this->preflight['affected_count'] ?? 0) <= 0) +
+ Nothing to do for the current scope. +
+ @endif + @else +
+ Run Preflight to see how many findings would change for the selected scope. +
+ @endif +
+
+
+ + diff --git a/resources/views/filament/system/pages/ops/runs.blade.php b/resources/views/filament/system/pages/ops/runs.blade.php new file mode 100644 index 0000000..be7f0a3 --- /dev/null +++ b/resources/views/filament/system/pages/ops/runs.blade.php @@ -0,0 +1,4 @@ + + {{ $this->table }} + + diff --git a/resources/views/filament/system/pages/ops/view-run.blade.php b/resources/views/filament/system/pages/ops/view-run.blade.php new file mode 100644 index 0000000..0d13aa5 --- /dev/null +++ b/resources/views/filament/system/pages/ops/view-run.blade.php @@ -0,0 +1,111 @@ +@php + /** @var \App\Models\OperationRun $run */ + $run = $this->run; + + $scope = (string) data_get($run->context, 'runbook.scope', 'unknown'); + $targetTenantId = data_get($run->context, 'runbook.target_tenant_id'); + $reasonCode = data_get($run->context, 'reason.reason_code'); + $reasonText = data_get($run->context, 'reason.reason_text'); + + $platformInitiator = data_get($run->context, 'platform_initiator', []); +@endphp + + +
+
+
+
+
+ Run #{{ (int) $run->getKey() }} +
+
+ {{ \App\Support\OperationCatalog::label((string) $run->type) }} +
+
+ +
+
Started: {{ $run->started_at?->toDayDateTimeString() ?? '—' }}
+
Completed: {{ $run->completed_at?->toDayDateTimeString() ?? '—' }}
+
+
+ +
+
+
Status
+
{{ (string) $run->status }}
+
+ +
+
Outcome
+
{{ (string) $run->outcome }}
+
+ +
+
Scope
+
+ @if ($scope === 'single_tenant') + Single tenant {{ is_numeric($targetTenantId) ? '#'.(int) $targetTenantId : '' }} + @elseif ($scope === 'all_tenants') + All tenants + @else + {{ $scope }} + @endif +
+
+
+ +
+
+
+ Initiator: + {{ (string) ($run->initiator_name ?? '—') }} +
+ + @if (is_array($platformInitiator) && ($platformInitiator['email'] ?? null)) +
+ Platform user: + {{ (string) ($platformInitiator['email'] ?? '') }} +
+ @endif +
+ + @if (is_string($reasonCode) && is_string($reasonText) && trim($reasonCode) !== '' && trim($reasonText) !== '') +
+
Reason
+
+ {{ $reasonCode }} + — + {{ $reasonText }} +
+
+ @endif +
+
+ + @if (! empty($run->summary_counts)) +
+
Summary counts
+
+ @include('filament.partials.json-viewer', ['value' => $run->summary_counts]) +
+
+ @endif + + @if (! empty($run->failure_summary)) +
+
Failures
+
+ @include('filament.partials.json-viewer', ['value' => $run->failure_summary]) +
+
+ @endif + +
+
Context
+
+ @include('filament.partials.json-viewer', ['value' => $run->context ?? []]) +
+
+
+ + diff --git a/specs/113-platform-ops-runbooks/tasks.md b/specs/113-platform-ops-runbooks/tasks.md index aa5235d..1da16ab 100644 --- a/specs/113-platform-ops-runbooks/tasks.md +++ b/specs/113-platform-ops-runbooks/tasks.md @@ -52,35 +52,35 @@ ## Phase 3: User Story 1 — Operator runs a runbook safely (Priority: P1) 🎯 ### Tests for User Story 1 -- [ ] T016 [P] [US1] Add runbook preflight tests in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillPreflightTest.php (single tenant + all tenants preflight returns affected_count) -- [ ] T017 [P] [US1] Add runbook start/confirmation tests in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillStartTest.php (typed confirmation + reason required for all_tenants; disabled when affected_count=0) -- [ ] T018 [P] [US1] Add break-glass reason enforcement + recording tests in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillBreakGlassTest.php (reason required when break-glass active; break-glass marker and reason recorded on run + audit) -- [ ] T019 [P] [US1] Add Ops-UX feedback contract test for start surface in tests/Feature/System/OpsRunbooks/OpsUxStartSurfaceContractTest.php (toast intent-only + “View run” link; no DB queued/running notifications) -- [ ] T020 [P] [US1] Add audit fail-safe test in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillAuditFailSafeTest.php (audit logger failure does not crash run; run still records failure outcome) +- [X] T016 [P] [US1] Add runbook preflight tests in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillPreflightTest.php (single tenant + all tenants preflight returns affected_count) +- [X] T017 [P] [US1] Add runbook start/confirmation tests in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillStartTest.php (typed confirmation + reason required for all_tenants; disabled when affected_count=0) +- [X] T018 [P] [US1] Add break-glass reason enforcement + recording tests in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillBreakGlassTest.php (reason required when break-glass active; break-glass marker and reason recorded on run + audit) +- [X] T019 [P] [US1] Add Ops-UX feedback contract test for start surface in tests/Feature/System/OpsRunbooks/OpsUxStartSurfaceContractTest.php (toast intent-only + “View run” link; no DB queued/running notifications) +- [X] T020 [P] [US1] Add audit fail-safe test in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillAuditFailSafeTest.php (audit logger failure does not crash run; run still records failure outcome) ### Implementation for User Story 1 -- [ ] T021 [US1] Create runbook service app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php with methods preflight(scope) and start(scope, initiator, reason, source) -- [ ] T022 [P] [US1] Create runbook scope/value objects in app/Services/Runbooks/FindingsLifecycleBackfillScope.php and app/Services/Runbooks/RunbookReason.php (validate reason_code and reason_text max 500 chars; include break-glass reason requirements) -- [ ] T023 [US1] Add audit events for preflight/start/completed/failed using AuditLogger in app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (action IDs per specs/113-platform-ops-runbooks/data-model.md; must be fail-safe) -- [ ] T024 [US1] Record break-glass marker + reason on OperationRun context and audit in app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (SR-005) +- [X] T021 [US1] Create runbook service app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php with methods preflight(scope) and start(scope, initiator, reason, source) +- [X] T022 [P] [US1] Create runbook scope/value objects in app/Services/Runbooks/FindingsLifecycleBackfillScope.php and app/Services/Runbooks/RunbookReason.php (validate reason_code and reason_text max 500 chars; include break-glass reason requirements) +- [X] T023 [US1] Add audit events for preflight/start/completed/failed using AuditLogger in app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (action IDs per specs/113-platform-ops-runbooks/data-model.md; must be fail-safe) +- [X] T024 [US1] Record break-glass marker + reason on OperationRun context and audit in app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (SR-005) -- [ ] T025 [US1] Implement all-tenants orchestration job in app/Jobs/BackfillFindingLifecycleWorkspaceJob.php (create/lock workspace-scoped OperationRun; dispatch tenant fan-out; set summary_counts[tenants/total/processed]) -- [ ] T026 [US1] Implement tenant worker job that updates the shared workspace run in app/Jobs/BackfillFindingLifecycleTenantIntoWorkspaceRunJob.php (chunk writes; increment summary_counts keys from OperationSummaryKeys::all(); append failures; call maybeCompleteBulkRun()) -- [ ] T027 [US1] Ensure scope-level lock prevents concurrent all-tenants runs in app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (lock key includes workspace + scope) +- [X] T025 [US1] Implement all-tenants orchestration job in app/Jobs/BackfillFindingLifecycleWorkspaceJob.php (create/lock workspace-scoped OperationRun; dispatch tenant fan-out; set summary_counts[tenants/total/processed]) +- [X] T026 [US1] Implement tenant worker job that updates the shared workspace run in app/Jobs/BackfillFindingLifecycleTenantIntoWorkspaceRunJob.php (chunk writes; increment summary_counts keys from OperationSummaryKeys::all(); append failures; call maybeCompleteBulkRun()) +- [X] T027 [US1] Ensure scope-level lock prevents concurrent all-tenants runs in app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (lock key includes workspace + scope) -- [ ] T028 [US1] Enable platform in-app notifications for run completion/failure by turning on database notifications in app/Providers/Filament/SystemPanelProvider.php (ensure terminal notification is OperationRunCompleted, initiator-only) -- [ ] T029 [P] [US1] Add System “View run” URL helper in app/Support/System/SystemOperationRunLinks.php and use it for UI + alerts/notifications (avoid admin-plane links) -- [ ] T030 [US1] Dispatch Alerts event on failure using app/Services/Alerts/AlertDispatchService.php from app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (event_type operations.run.failed; include System “View run” URL) +- [X] T028 [US1] Enable platform in-app notifications for run completion/failure by turning on database notifications in app/Providers/Filament/SystemPanelProvider.php (ensure terminal notification is OperationRunCompleted, initiator-only) +- [X] T029 [P] [US1] Add System “View run” URL helper in app/Support/System/SystemOperationRunLinks.php and use it for UI + alerts/notifications (avoid admin-plane links) +- [X] T030 [US1] Dispatch Alerts event on failure using app/Services/Alerts/AlertDispatchService.php from app/Services/Runbooks/FindingsLifecycleBackfillRunbookService.php (event_type operations.run.failed; include System “View run” URL) -- [ ] T031 [US1] Create System runbooks page class app/Filament/System/Pages/Ops/Runbooks.php (capability-gated; scope selector uses AllowedTenantUniverse; Preflight action; Run action with confirmation + typed confirm + reason) -- [ ] T032 [P] [US1] Create System runbooks page view resources/views/filament/system/pages/ops/runbooks.blade.php (operator warning; show preflight results + disable Run when nothing to do) +- [X] T031 [US1] Create System runbooks page class app/Filament/System/Pages/Ops/Runbooks.php (capability-gated; scope selector uses AllowedTenantUniverse; Preflight action; Run action with confirmation + typed confirm + reason) +- [X] T032 [P] [US1] Create System runbooks page view resources/views/filament/system/pages/ops/runbooks.blade.php (operator warning; show preflight results + disable Run when nothing to do) -- [ ] T033 [US1] Create System runs list page class app/Filament/System/Pages/Ops/Runs.php (table listing operation runs for runbook types; default sort newest) -- [ ] T034 [P] [US1] Create System runs list view resources/views/filament/system/pages/ops/runs.blade.php (record inspection affordance: clickable row -> run detail) +- [X] T033 [US1] Create System runs list page class app/Filament/System/Pages/Ops/Runs.php (table listing operation runs for runbook types; default sort newest) +- [X] T034 [P] [US1] Create System runs list view resources/views/filament/system/pages/ops/runs.blade.php (record inspection affordance: clickable row -> run detail) -- [ ] T035 [US1] Create System run detail page class app/Filament/System/Pages/Ops/ViewRun.php (infolist rendering of OperationRun; show scope/actor/counts/failures) -- [ ] T036 [P] [US1] Create System run detail view resources/views/filament/system/pages/ops/view-run.blade.php +- [X] T035 [US1] Create System run detail page class app/Filament/System/Pages/Ops/ViewRun.php (infolist rendering of OperationRun; show scope/actor/counts/failures) +- [X] T036 [P] [US1] Create System run detail view resources/views/filament/system/pages/ops/view-run.blade.php --- @@ -92,13 +92,13 @@ ## Phase 4: User Story 2 — Customers never see maintenance actions (Priority: ### Tests for User Story 2 -- [ ] T037 [P] [US2] Add regression test asserting /admin Findings list has no backfill action by default in tests/Feature/Filament/Spec113/AdminFindingsNoMaintenanceActionsTest.php (targets app/Filament/Resources/FindingResource/Pages/ListFindings.php) -- [ ] T038 [P] [US2] Add tenant-plane 404 test for /system/ops/runbooks in tests/Feature/System/Spec113/TenantPlaneCannotAccessSystemTest.php +- [X] T037 [P] [US2] Add regression test asserting /admin Findings list has no backfill action by default in tests/Feature/Filament/Spec113/AdminFindingsNoMaintenanceActionsTest.php (targets app/Filament/Resources/FindingResource/Pages/ListFindings.php) +- [X] T038 [P] [US2] Add tenant-plane 404 test for /system/ops/runbooks in tests/Feature/System/Spec113/TenantPlaneCannotAccessSystemTest.php ### Implementation for User Story 2 -- [ ] T039 [US2] Remove or feature-flag off the legacy header action in app/Filament/Resources/FindingResource/Pages/ListFindings.php (FR-001; default off in production-like envs) -- [ ] T040 [US2] Add a config-backed feature flag defaulting to false in config/tenantpilot.php (e.g., allow_admin_maintenance_actions) and wire it in app/Filament/Resources/FindingResource/Pages/ListFindings.php +- [X] T039 [US2] Remove or feature-flag off the legacy header action in app/Filament/Resources/FindingResource/Pages/ListFindings.php (FR-001; default off in production-like envs) +- [X] T040 [US2] Add a config-backed feature flag defaulting to false in config/tenantpilot.php (e.g., allow_admin_maintenance_actions) and wire it in app/Filament/Resources/FindingResource/Pages/ListFindings.php --- @@ -110,23 +110,23 @@ ## Phase 5: User Story 3 — Same logic for deploy-time and operator re-run (Pri ### Tests for User Story 3 -- [ ] T041 [P] [US3] Add idempotency test in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillIdempotencyTest.php (second run updated=0 and/or preflight affected_count=0) -- [ ] T042 [P] [US3] Add deploy-time entry point test in tests/Feature/Console/Spec113/DeployRunbooksCommandTest.php (command delegates to FindingsLifecycleBackfillRunbookService) +- [X] T041 [P] [US3] Add idempotency test in tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillIdempotencyTest.php (second run updated=0 and/or preflight affected_count=0) +- [X] T042 [P] [US3] Add deploy-time entry point test in tests/Feature/Console/Spec113/DeployRunbooksCommandTest.php (command delegates to FindingsLifecycleBackfillRunbookService) ### Implementation for User Story 3 -- [ ] T043 [US3] Refactor CLI command to call shared runbook service in app/Console/Commands/TenantpilotBackfillFindingLifecycle.php (single-tenant scope, source=cli) -- [ ] T044 [US3] Add deploy-time runbooks command in app/Console/Commands/TenantpilotRunDeployRunbooks.php (source=deploy_hook; initiator null; uses FindingsLifecycleBackfillRunbookService) -- [ ] T045 [US3] Ensure System UI uses the same runbook service start() call path in app/Filament/System/Pages/Ops/Runbooks.php (source=system_ui) -- [ ] T046 [US3] Ensure initiator-null runs do not emit terminal DB notification in app/Services/OperationRunService.php (system-run behavior; audit/alerts still apply) +- [X] T043 [US3] Refactor CLI command to call shared runbook service in app/Console/Commands/TenantpilotBackfillFindingLifecycle.php (single-tenant scope, source=cli) +- [X] T044 [US3] Add deploy-time runbooks command in app/Console/Commands/TenantpilotRunDeployRunbooks.php (source=deploy_hook; initiator null; uses FindingsLifecycleBackfillRunbookService) +- [X] T045 [US3] Ensure System UI uses the same runbook service start() call path in app/Filament/System/Pages/Ops/Runbooks.php (source=system_ui) +- [X] T046 [US3] Ensure initiator-null runs do not emit terminal DB notification in app/Services/OperationRunService.php (system-run behavior; audit/alerts still apply) --- ## Phase 6: Polish & Cross-Cutting Concerns -- [ ] T047 [P] Run new Spec 113 tests via vendor/bin/sail artisan test --compact tests/Feature/System/Spec113/ (ensure all new tests pass) -- [ ] T048 [P] Run Ops Runbooks tests via vendor/bin/sail artisan test --compact tests/Feature/System/OpsRunbooks/ (ensure US1/US3 tests pass) -- [ ] T049 [P] Run formatting on touched files via vendor/bin/sail bin pint --dirty --format agent (targets app/Http/Middleware/, app/Filament/System/Pages/, app/Services/Runbooks/, tests/Feature/System/) +- [X] T047 [P] Run new Spec 113 tests via vendor/bin/sail artisan test --compact tests/Feature/System/Spec113/ (ensure all new tests pass) +- [X] T048 [P] Run Ops Runbooks tests via vendor/bin/sail artisan test --compact tests/Feature/System/OpsRunbooks/ (ensure US1/US3 tests pass) +- [X] T049 [P] Run formatting on touched files via vendor/bin/sail bin pint --dirty --format agent (targets app/Http/Middleware/, app/Filament/System/Pages/, app/Services/Runbooks/, tests/Feature/System/) --- diff --git a/tests/Feature/Console/Spec113/DeployRunbooksCommandTest.php b/tests/Feature/Console/Spec113/DeployRunbooksCommandTest.php new file mode 100644 index 0000000..f6cc1ab --- /dev/null +++ b/tests/Feature/Console/Spec113/DeployRunbooksCommandTest.php @@ -0,0 +1,31 @@ +create(); + + $this->mock(FindingsLifecycleBackfillRunbookService::class, function ($mock) use ($run): void { + $mock->shouldReceive('start') + ->once() + ->withArgs(function ($scope, $initiator, $reason, $source): bool { + return $scope instanceof FindingsLifecycleBackfillScope + && $scope->isAllTenants() + && $initiator === null + && $reason instanceof RunbookReason + && $source === 'deploy_hook'; + }) + ->andReturn($run); + }); + + $this->artisan('tenantpilot:run-deploy-runbooks') + ->assertExitCode(0); +}); diff --git a/tests/Feature/Filament/Spec113/AdminFindingsNoMaintenanceActionsTest.php b/tests/Feature/Filament/Spec113/AdminFindingsNoMaintenanceActionsTest.php new file mode 100644 index 0000000..5309f92 --- /dev/null +++ b/tests/Feature/Filament/Spec113/AdminFindingsNoMaintenanceActionsTest.php @@ -0,0 +1,20 @@ +actingAs($user); + Filament::setTenant($tenant, true); + + Livewire::test(ListFindings::class) + ->assertActionDoesNotExist('backfill_lifecycle'); +}); diff --git a/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillAuditFailSafeTest.php b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillAuditFailSafeTest.php new file mode 100644 index 0000000..7b5f87e --- /dev/null +++ b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillAuditFailSafeTest.php @@ -0,0 +1,87 @@ +create([ + 'tenant_id' => null, + 'external_id' => 'platform', + 'name' => 'Platform', + ]); +}); + +it('does not crash when audit logging fails and still finalizes a failed run', function () { + $this->mock(AuditLogger::class, function ($mock): void { + $mock->shouldReceive('log')->andThrow(new RuntimeException('audit unavailable')); + }); + + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'due_at' => null, + ]); + + $user = PlatformUser::factory()->create([ + 'capabilities' => [ + PlatformCapabilities::ACCESS_SYSTEM_PANEL, + PlatformCapabilities::OPS_VIEW, + PlatformCapabilities::RUNBOOKS_VIEW, + PlatformCapabilities::RUNBOOKS_RUN, + PlatformCapabilities::RUNBOOKS_FINDINGS_LIFECYCLE_BACKFILL, + ], + 'is_active' => true, + ]); + + $this->actingAs($user, 'platform'); + + $runbook = app(FindingsLifecycleBackfillRunbookService::class); + + $run = $runbook->start( + scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), + initiator: $user, + reason: null, + source: 'system_ui', + ); + + $runs = app(OperationRunService::class); + + $runs->updateRun( + $run, + status: 'completed', + outcome: 'failed', + failures: [ + [ + 'code' => 'test.failed', + 'message' => 'Forced failure for audit fail-safe test.', + ], + ], + ); + + $runbook->maybeFinalize($run); + + $run->refresh(); + + expect($run->status)->toBe('completed'); + expect($run->outcome)->toBe('failed'); +}); diff --git a/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillBreakGlassTest.php b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillBreakGlassTest.php new file mode 100644 index 0000000..8d2dc27 --- /dev/null +++ b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillBreakGlassTest.php @@ -0,0 +1,111 @@ +create([ + 'tenant_id' => null, + 'external_id' => 'platform', + 'name' => 'Platform', + ]); + + config()->set('tenantpilot.break_glass.enabled', true); + config()->set('tenantpilot.break_glass.ttl_minutes', 15); +}); + +it('requires a reason when break-glass is active and records break-glass on the run + audit', function () { + Queue::fake(); + + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $customerTenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $customerTenant->getKey(), + 'due_at' => null, + ]); + + $user = PlatformUser::factory()->create([ + 'capabilities' => [ + PlatformCapabilities::ACCESS_SYSTEM_PANEL, + PlatformCapabilities::OPS_VIEW, + PlatformCapabilities::RUNBOOKS_VIEW, + PlatformCapabilities::RUNBOOKS_RUN, + PlatformCapabilities::RUNBOOKS_FINDINGS_LIFECYCLE_BACKFILL, + PlatformCapabilities::USE_BREAK_GLASS, + ], + 'is_active' => true, + ]); + + $this->actingAs($user, 'platform'); + + Livewire::test(Dashboard::class) + ->callAction('enter_break_glass', data: [ + 'reason' => 'Recovery test', + ]) + ->assertHasNoActionErrors(); + + Livewire::test(Runbooks::class) + ->callAction('preflight', data: [ + 'scope_mode' => FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT, + 'tenant_id' => (int) $customerTenant->getKey(), + ]) + ->assertSet('preflight.affected_count', 1) + ->callAction('run', data: []) + ->assertHasActionErrors(['reason_code', 'reason_text']); + + Livewire::test(Runbooks::class) + ->callAction('preflight', data: [ + 'scope_mode' => FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT, + 'tenant_id' => (int) $customerTenant->getKey(), + ]) + ->assertSet('preflight.affected_count', 1) + ->callAction('run', data: [ + 'reason_code' => 'INCIDENT', + 'reason_text' => 'Break-glass backfill required', + ]) + ->assertHasNoActionErrors() + ->assertNotified(); + + $run = OperationRun::query() + ->where('type', 'findings.lifecycle.backfill') + ->latest('id') + ->first(); + + expect($run)->not->toBeNull(); + expect((int) $run?->tenant_id)->toBe((int) $customerTenant->getKey()); + expect(data_get($run?->context, 'platform_initiator.is_break_glass'))->toBeTrue(); + expect(data_get($run?->context, 'reason.reason_code'))->toBe('INCIDENT'); + expect(data_get($run?->context, 'reason.reason_text'))->toBe('Break-glass backfill required'); + + $audit = AuditLog::query() + ->where('action', 'platform.ops.runbooks.start') + ->latest('id') + ->first(); + + expect($audit)->not->toBeNull(); + expect($audit?->metadata['is_break_glass'] ?? null)->toBe(true); + expect($audit?->metadata['reason_code'] ?? null)->toBe('INCIDENT'); + expect($audit?->metadata['reason_text'] ?? null)->toBe('Break-glass backfill required'); +}); diff --git a/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillIdempotencyTest.php b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillIdempotencyTest.php new file mode 100644 index 0000000..645a695 --- /dev/null +++ b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillIdempotencyTest.php @@ -0,0 +1,78 @@ +create([ + 'tenant_id' => null, + 'external_id' => 'platform', + 'name' => 'Platform', + ]); +}); + +it('is idempotent: after a successful run, preflight reports nothing to do', function () { + Queue::fake(); + + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'due_at' => null, + ]); + + $runbook = app(FindingsLifecycleBackfillRunbookService::class); + + $initial = $runbook->preflight(FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey())); + + expect($initial['affected_count'])->toBe(1); + + $runbook->start( + scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), + initiator: null, + reason: null, + source: 'system_ui', + ); + + $job = new BackfillFindingLifecycleJob( + tenantId: (int) $tenant->getKey(), + workspaceId: (int) $tenant->workspace_id, + initiatorUserId: null, + ); + + $job->handle( + app(OperationRunService::class), + app(\App\Services\Findings\FindingSlaPolicy::class), + $runbook, + ); + + $after = $runbook->preflight(FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey())); + + expect($after['affected_count'])->toBe(0); + + expect(fn () => $runbook->start( + scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), + initiator: null, + reason: null, + source: 'system_ui', + ))->toThrow(ValidationException::class); +}); diff --git a/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillPreflightTest.php b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillPreflightTest.php new file mode 100644 index 0000000..5f4308d --- /dev/null +++ b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillPreflightTest.php @@ -0,0 +1,84 @@ +create([ + 'tenant_id' => null, + 'external_id' => 'platform', + 'name' => 'Platform', + ]); +}); + +it('computes single-tenant preflight counts', function () { + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'due_at' => null, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + ]); + + $service = app(FindingsLifecycleBackfillRunbookService::class); + + $result = $service->preflight(FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey())); + + expect($result['total_count'])->toBe(2); + expect($result['affected_count'])->toBe(1); +}); + +it('computes all-tenants preflight counts scoped to the platform workspace', function () { + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenantA = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + $tenantB = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + $otherTenant = Tenant::factory()->create(); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenantA->getKey(), + 'due_at' => null, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenantB->getKey(), + 'sla_days' => null, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $otherTenant->getKey(), + 'due_at' => null, + ]); + + $service = app(FindingsLifecycleBackfillRunbookService::class); + + $result = $service->preflight(FindingsLifecycleBackfillScope::allTenants()); + + expect($result['estimated_tenants'])->toBe(2); + expect($result['total_count'])->toBe(2); + expect($result['affected_count'])->toBe(2); +}); diff --git a/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillStartTest.php b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillStartTest.php new file mode 100644 index 0000000..886a54a --- /dev/null +++ b/tests/Feature/System/OpsRunbooks/FindingsLifecycleBackfillStartTest.php @@ -0,0 +1,113 @@ +create([ + 'tenant_id' => null, + 'external_id' => 'platform', + 'name' => 'Platform', + ]); +}); + +it('disables running when preflight indicates nothing to do', function () { + Queue::fake(); + + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + ]); + + $user = PlatformUser::factory()->create([ + 'capabilities' => [ + PlatformCapabilities::ACCESS_SYSTEM_PANEL, + PlatformCapabilities::OPS_VIEW, + PlatformCapabilities::RUNBOOKS_VIEW, + PlatformCapabilities::RUNBOOKS_RUN, + PlatformCapabilities::RUNBOOKS_FINDINGS_LIFECYCLE_BACKFILL, + ], + 'is_active' => true, + ]); + + $this->actingAs($user, 'platform'); + + Livewire::test(Runbooks::class) + ->callAction('preflight', data: [ + 'scope_mode' => FindingsLifecycleBackfillScope::MODE_ALL_TENANTS, + ]) + ->assertSet('preflight.affected_count', 0) + ->assertActionDisabled('run'); +}); + +it('requires typed confirmation and a reason for all-tenants runs', function () { + Queue::fake(); + + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'due_at' => null, + ]); + + $user = PlatformUser::factory()->create([ + 'capabilities' => [ + PlatformCapabilities::ACCESS_SYSTEM_PANEL, + PlatformCapabilities::OPS_VIEW, + PlatformCapabilities::RUNBOOKS_VIEW, + PlatformCapabilities::RUNBOOKS_RUN, + PlatformCapabilities::RUNBOOKS_FINDINGS_LIFECYCLE_BACKFILL, + ], + 'is_active' => true, + ]); + + $this->actingAs($user, 'platform'); + + Livewire::test(Runbooks::class) + ->callAction('preflight', data: [ + 'scope_mode' => FindingsLifecycleBackfillScope::MODE_ALL_TENANTS, + ]) + ->assertSet('preflight.affected_count', 1) + ->callAction('run', data: []) + ->assertHasActionErrors([ + 'typed_confirmation', + 'reason_code', + 'reason_text', + ]); + + Livewire::test(Runbooks::class) + ->callAction('preflight', data: [ + 'scope_mode' => FindingsLifecycleBackfillScope::MODE_ALL_TENANTS, + ]) + ->assertSet('preflight.affected_count', 1) + ->callAction('run', data: [ + 'typed_confirmation' => 'backfill', + 'reason_code' => 'DATA_REPAIR', + 'reason_text' => 'Test run', + ]) + ->assertHasActionErrors(['typed_confirmation']); +}); diff --git a/tests/Feature/System/OpsRunbooks/OpsUxStartSurfaceContractTest.php b/tests/Feature/System/OpsRunbooks/OpsUxStartSurfaceContractTest.php new file mode 100644 index 0000000..76d79ab --- /dev/null +++ b/tests/Feature/System/OpsRunbooks/OpsUxStartSurfaceContractTest.php @@ -0,0 +1,89 @@ +create([ + 'tenant_id' => null, + 'external_id' => 'platform', + 'name' => 'Platform', + ]); +}); + +it('uses an intent-only toast with a working view-run link and does not emit queued database notifications', function () { + Queue::fake(); + NotificationFacade::fake(); + + $platformTenant = Tenant::query()->where('external_id', 'platform')->firstOrFail(); + + $tenant = Tenant::factory()->create([ + 'workspace_id' => (int) $platformTenant->workspace_id, + ]); + + Finding::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'due_at' => null, + ]); + + $user = PlatformUser::factory()->create([ + 'capabilities' => [ + PlatformCapabilities::ACCESS_SYSTEM_PANEL, + PlatformCapabilities::OPS_VIEW, + PlatformCapabilities::RUNBOOKS_VIEW, + PlatformCapabilities::RUNBOOKS_RUN, + PlatformCapabilities::RUNBOOKS_FINDINGS_LIFECYCLE_BACKFILL, + ], + 'is_active' => true, + ]); + + $this->actingAs($user, 'platform'); + + Livewire::test(Runbooks::class) + ->callAction('preflight', data: [ + 'scope_mode' => FindingsLifecycleBackfillScope::MODE_ALL_TENANTS, + ]) + ->assertSet('preflight.affected_count', 1) + ->callAction('run', data: [ + 'typed_confirmation' => 'BACKFILL', + 'reason_code' => 'DATA_REPAIR', + 'reason_text' => 'Operator test', + ]) + ->assertHasNoActionErrors() + ->assertNotified('Findings lifecycle backfill queued'); + + NotificationFacade::assertNothingSent(); + expect(DatabaseNotification::query()->count())->toBe(0); + + $run = OperationRun::query() + ->where('type', 'findings.lifecycle.backfill') + ->latest('id') + ->first(); + + expect($run)->not->toBeNull(); + + $viewUrl = SystemOperationRunLinks::view($run); + + $this->get($viewUrl) + ->assertSuccessful() + ->assertSee('Run #'.(int) $run?->getKey()); +}); diff --git a/tests/Feature/System/Spec113/TenantPlaneCannotAccessSystemTest.php b/tests/Feature/System/Spec113/TenantPlaneCannotAccessSystemTest.php new file mode 100644 index 0000000..601abde --- /dev/null +++ b/tests/Feature/System/Spec113/TenantPlaneCannotAccessSystemTest.php @@ -0,0 +1,16 @@ +create(); + + $this->actingAs($user) + ->get('/system/ops/runbooks') + ->assertNotFound(); +});