computePreflight($scope); $this->auditSafely( action: 'platform.ops.runbooks.preflight', scope: $scope, operationRunId: null, context: [ 'preflight' => $result, ], ); return $result; } public function start( FindingsLifecycleBackfillScope $scope, ?PlatformUser $initiator, ?RunbookReason $reason, string $source, ): OperationRun { $source = trim($source); if ($source === '') { throw ValidationException::withMessages([ 'source' => 'A run source is required.', ]); } $isBreakGlassActive = $this->breakGlassSession->isActive(); if ($scope->isAllTenants() || $isBreakGlassActive) { if (! $reason instanceof RunbookReason) { throw ValidationException::withMessages([ 'reason' => 'A reason is required for this run.', ]); } } $preflight = $this->computePreflight($scope); if (($preflight['affected_count'] ?? 0) <= 0) { throw ValidationException::withMessages([ 'preflight.affected_count' => 'Nothing to do for this scope.', ]); } $platformTenant = $this->platformTenant(); $workspace = $platformTenant->workspace; if (! $workspace instanceof Workspace) { throw new \RuntimeException('Platform tenant is missing its workspace.'); } if ($scope->isAllTenants()) { $lockKey = sprintf('tenantpilot:runbooks:%s:workspace:%d', self::RUNBOOK_KEY, (int) $workspace->getKey()); $lock = Cache::lock($lockKey, 900); if (! $lock->get()) { throw ValidationException::withMessages([ 'scope' => 'Another run is already in progress for this scope.', ]); } try { return $this->startAllTenants( workspace: $workspace, initiator: $initiator, reason: $reason, preflight: $preflight, source: $source, isBreakGlassActive: $isBreakGlassActive, ); } finally { $lock->release(); } } return $this->startSingleTenant( tenantId: (int) $scope->tenantId, initiator: $initiator, reason: $reason, preflight: $preflight, source: $source, isBreakGlassActive: $isBreakGlassActive, ); } public function maybeFinalize(OperationRun $run): void { $run->refresh(); if ($run->status !== OperationRunStatus::Completed->value) { return; } $context = is_array($run->context) ? $run->context : []; if ((string) data_get($context, 'runbook.key') !== self::RUNBOOK_KEY) { return; } $lockKey = sprintf('tenantpilot:runbooks:%s:finalize:%d', self::RUNBOOK_KEY, (int) $run->getKey()); $lock = Cache::lock($lockKey, 86400); if (! $lock->get()) { return; } try { $this->auditSafely( action: $run->outcome === OperationRunOutcome::Failed->value ? 'platform.ops.runbooks.failed' : 'platform.ops.runbooks.completed', scope: $this->scopeFromRunContext($context), operationRunId: (int) $run->getKey(), context: [ 'status' => (string) $run->status, 'outcome' => (string) $run->outcome, 'is_break_glass' => (bool) data_get($context, 'platform_initiator.is_break_glass', false), 'reason_code' => data_get($context, 'reason.reason_code'), 'reason_text' => data_get($context, 'reason.reason_text'), ], ); $this->notifyInitiatorSafely($run); if ($run->outcome === OperationRunOutcome::Failed->value) { $this->dispatchFailureAlertSafely($run); } } finally { $lock->release(); } } /** * @return array{affected_count: int, total_count: int, estimated_tenants?: int|null} */ private function computePreflight(FindingsLifecycleBackfillScope $scope): array { if ($scope->isSingleTenant()) { $tenant = Tenant::query()->whereKey((int) $scope->tenantId)->firstOrFail(); $this->allowedTenantUniverse->ensureAllowed($tenant); return $this->computeTenantPreflight($tenant); } $platformTenant = $this->platformTenant(); $workspaceId = (int) ($platformTenant->workspace_id ?? 0); $tenants = $this->allowedTenantUniverse ->query() ->where('workspace_id', $workspaceId) ->orderBy('id') ->get(); $affected = 0; $total = 0; foreach ($tenants as $tenant) { if (! $tenant instanceof Tenant) { continue; } $counts = $this->computeTenantPreflight($tenant); $affected += (int) ($counts['affected_count'] ?? 0); $total += (int) ($counts['total_count'] ?? 0); } return [ 'affected_count' => $affected, 'total_count' => $total, 'estimated_tenants' => $tenants->count(), ]; } /** * @return array{affected_count: int, total_count: int} */ private function computeTenantPreflight(Tenant $tenant): array { $query = Finding::query()->where('tenant_id', (int) $tenant->getKey()); $total = (int) (clone $query)->count(); $affected = 0; (clone $query) ->orderBy('id') ->chunkById(500, function ($findings) use (&$affected): void { foreach ($findings as $finding) { if (! $finding instanceof Finding) { continue; } if ($this->findingNeedsBackfill($finding)) { $affected++; } } }); $affected += $this->countDriftDuplicateConsolidations($tenant); return [ 'affected_count' => $affected, 'total_count' => $total, ]; } private function findingNeedsBackfill(Finding $finding): bool { if ($finding->first_seen_at === null || $finding->last_seen_at === null) { return true; } if ($finding->last_seen_at !== null && $finding->first_seen_at !== null) { if ($finding->last_seen_at->lt($finding->first_seen_at)) { return true; } } $timesSeen = is_numeric($finding->times_seen) ? (int) $finding->times_seen : 0; if ($timesSeen < 1) { return true; } if ($finding->status === Finding::STATUS_ACKNOWLEDGED) { return true; } if (Finding::isOpenStatus((string) $finding->status)) { if ($finding->sla_days === null || $finding->due_at === null) { return true; } } if ($finding->finding_type === Finding::FINDING_TYPE_DRIFT) { $recurrenceKey = $finding->recurrence_key !== null ? trim((string) $finding->recurrence_key) : ''; if ($recurrenceKey === '') { $scopeKey = trim((string) ($finding->scope_key ?? '')); $subjectType = trim((string) ($finding->subject_type ?? '')); $subjectExternalId = trim((string) ($finding->subject_external_id ?? '')); if ($scopeKey !== '' && $subjectType !== '' && $subjectExternalId !== '') { $evidence = is_array($finding->evidence_jsonb) ? $finding->evidence_jsonb : []; $kind = data_get($evidence, 'summary.kind'); if (is_string($kind) && trim($kind) !== '') { return true; } } } } return false; } private function countDriftDuplicateConsolidations(Tenant $tenant): int { $rows = Finding::query() ->where('tenant_id', (int) $tenant->getKey()) ->where('finding_type', Finding::FINDING_TYPE_DRIFT) ->whereNotNull('recurrence_key') ->select(['recurrence_key', DB::raw('COUNT(*) as count')]) ->groupBy('recurrence_key') ->havingRaw('COUNT(*) > 1') ->get(); $duplicates = 0; foreach ($rows as $row) { $count = is_numeric($row->count ?? null) ? (int) $row->count : 0; if ($count > 1) { $duplicates += ($count - 1); } } return $duplicates; } private function startAllTenants( Workspace $workspace, ?PlatformUser $initiator, ?RunbookReason $reason, array $preflight, string $source, bool $isBreakGlassActive, ): OperationRun { $run = $this->operationRunService->ensureWorkspaceRunWithIdentity( workspace: $workspace, type: self::RUNBOOK_KEY, identityInputs: [ 'runbook' => self::RUNBOOK_KEY, 'scope' => FindingsLifecycleBackfillScope::MODE_ALL_TENANTS, ], context: $this->buildRunContext( workspaceId: (int) $workspace->getKey(), scope: FindingsLifecycleBackfillScope::allTenants(), initiator: $initiator, reason: $reason, preflight: $preflight, source: $source, isBreakGlassActive: $isBreakGlassActive, ), initiator: null, ); if ($initiator instanceof PlatformUser && $run->wasRecentlyCreated) { $run->update(['initiator_name' => $initiator->name ?: $initiator->email]); $run->refresh(); } $this->auditSafely( action: 'platform.ops.runbooks.start', scope: FindingsLifecycleBackfillScope::allTenants(), operationRunId: (int) $run->getKey(), context: [ 'preflight' => $preflight, 'is_break_glass' => $isBreakGlassActive, ] + ($reason instanceof RunbookReason ? $reason->toArray() : []), ); if (! $run->wasRecentlyCreated) { return $run; } $this->operationRunService->dispatchOrFail($run, function () use ($run, $workspace): void { BackfillFindingLifecycleWorkspaceJob::dispatch( operationRunId: (int) $run->getKey(), workspaceId: (int) $workspace->getKey(), ); }); return $run; } private function startSingleTenant( int $tenantId, ?PlatformUser $initiator, ?RunbookReason $reason, array $preflight, string $source, bool $isBreakGlassActive, ): OperationRun { $tenant = Tenant::query()->whereKey($tenantId)->firstOrFail(); $this->allowedTenantUniverse->ensureAllowed($tenant); $run = $this->operationRunService->ensureRunWithIdentity( tenant: $tenant, type: self::RUNBOOK_KEY, identityInputs: [ 'tenant_id' => (int) $tenant->getKey(), 'trigger' => 'backfill', ], context: $this->buildRunContext( workspaceId: (int) $tenant->workspace_id, scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), initiator: $initiator, reason: $reason, preflight: $preflight, source: $source, isBreakGlassActive: $isBreakGlassActive, ), initiator: null, ); if ($initiator instanceof PlatformUser && $run->wasRecentlyCreated) { $run->update(['initiator_name' => $initiator->name ?: $initiator->email]); $run->refresh(); } $this->auditSafely( action: 'platform.ops.runbooks.start', scope: FindingsLifecycleBackfillScope::singleTenant((int) $tenant->getKey()), operationRunId: (int) $run->getKey(), context: [ 'preflight' => $preflight, 'is_break_glass' => $isBreakGlassActive, ] + ($reason instanceof RunbookReason ? $reason->toArray() : []), ); if (! $run->wasRecentlyCreated) { return $run; } $this->operationRunService->dispatchOrFail($run, function () use ($tenant): void { BackfillFindingLifecycleJob::dispatch( tenantId: (int) $tenant->getKey(), workspaceId: (int) $tenant->workspace_id, initiatorUserId: null, ); }); return $run; } private function platformTenant(): Tenant { $tenant = Tenant::query()->where('external_id', 'platform')->first(); if (! $tenant instanceof Tenant) { throw new \RuntimeException('Platform tenant is missing.'); } return $tenant; } /** * @return array */ private function buildRunContext( int $workspaceId, FindingsLifecycleBackfillScope $scope, ?PlatformUser $initiator, ?RunbookReason $reason, array $preflight, string $source, bool $isBreakGlassActive, ): array { $context = [ 'workspace_id' => $workspaceId, 'runbook' => [ 'key' => self::RUNBOOK_KEY, 'scope' => $scope->mode, 'target_tenant_id' => $scope->tenantId, 'source' => $source, ], 'preflight' => [ 'affected_count' => (int) ($preflight['affected_count'] ?? 0), 'total_count' => (int) ($preflight['total_count'] ?? 0), 'estimated_tenants' => $preflight['estimated_tenants'] ?? null, ], ]; if ($reason instanceof RunbookReason) { $context['reason'] = $reason->toArray(); } if ($initiator instanceof PlatformUser) { $context['platform_initiator'] = [ 'platform_user_id' => (int) $initiator->getKey(), 'email' => (string) $initiator->email, 'name' => (string) $initiator->name, 'is_break_glass' => $isBreakGlassActive, ]; } return $context; } private function scopeFromRunContext(array $context): FindingsLifecycleBackfillScope { $scope = data_get($context, 'runbook.scope'); $tenantId = data_get($context, 'runbook.target_tenant_id'); if ($scope === FindingsLifecycleBackfillScope::MODE_SINGLE_TENANT && is_numeric($tenantId)) { return FindingsLifecycleBackfillScope::singleTenant((int) $tenantId); } return FindingsLifecycleBackfillScope::allTenants(); } /** * @param array $context */ private function auditSafely( string $action, FindingsLifecycleBackfillScope $scope, ?int $operationRunId, array $context = [], ): void { try { $platformTenant = $this->platformTenant(); $actor = auth('platform')->user(); $actorId = null; $actorEmail = null; $actorName = null; if ($actor instanceof PlatformUser) { $actorId = (int) $actor->getKey(); $actorEmail = (string) $actor->email; $actorName = (string) $actor->name; } $metadata = [ 'runbook_key' => self::RUNBOOK_KEY, 'scope' => $scope->mode, 'target_tenant_id' => $scope->tenantId, 'operation_run_id' => $operationRunId, 'ip' => request()->ip(), 'user_agent' => request()->userAgent(), ]; $this->auditLogger->log( tenant: $platformTenant, action: $action, context: [ 'metadata' => array_filter($metadata, static fn (mixed $value): bool => $value !== null), ] + $context, actorId: $actorId, actorEmail: $actorEmail, actorName: $actorName, status: 'success', resourceType: 'operation_run', resourceId: $operationRunId !== null ? (string) $operationRunId : null, ); } catch (Throwable) { // Audit is fail-safe (must not crash runbooks). } } private function notifyInitiatorSafely(OperationRun $run): void { try { $platformUserId = data_get($run->context, 'platform_initiator.platform_user_id'); if (! is_numeric($platformUserId)) { return; } $platformUser = PlatformUser::query()->whereKey((int) $platformUserId)->first(); if (! $platformUser instanceof PlatformUser) { return; } $platformUser->notify(new OperationRunCompleted($run)); } catch (Throwable) { // Notifications must not crash the runbook. } } private function dispatchFailureAlertSafely(OperationRun $run): void { try { $platformTenant = $this->platformTenant(); $workspace = $platformTenant->workspace; if (! $workspace instanceof Workspace) { return; } $this->alertDispatchService->dispatchEvent($workspace, [ 'tenant_id' => (int) $platformTenant->getKey(), 'event_type' => 'operations.run.failed', 'severity' => 'high', 'title' => 'Operation failed: Findings lifecycle backfill', 'body' => 'A findings lifecycle backfill run failed.', 'metadata' => [ 'operation_run_id' => (int) $run->getKey(), 'operation_type' => (string) $run->type, 'scope' => (string) data_get($run->context, 'runbook.scope', ''), 'view_run_url' => SystemOperationRunLinks::view($run), ], ]); } catch (Throwable) { // Alerts must not crash the runbook. } } }