operationRun = $operationRun; } public function middleware(): array { return [new TrackOperationRun]; } public function handle( PolicySyncService $policySyncService, BackupService $backupService, PolicyTypeResolver $policyTypeResolver, ScheduleTimeService $scheduleTimeService, AuditLogger $auditLogger, RunErrorMapper $errorMapper, ): void { $run = BackupScheduleRun::query() ->with(['schedule', 'tenant', 'user']) ->find($this->backupScheduleRunId); if (! $run) { if ($this->operationRun) { $this->markOperationRunFailed( run: $this->operationRun, summaryCounts: [], reasonCode: 'run_not_found', reason: 'Backup schedule run not found.', ); } return; } $tenant = $run->tenant; if ($tenant instanceof Tenant) { $this->resolveOperationRunFromContext($tenant, $run); } if ($this->operationRun) { $this->operationRun->update([ 'context' => array_merge($this->operationRun->context ?? [], [ 'backup_schedule_id' => (int) $run->backup_schedule_id, 'backup_schedule_run_id' => (int) $run->getKey(), ]), ]); /** @var OperationRunService $operationRunService */ $operationRunService = app(OperationRunService::class); if ($this->operationRun->status === 'queued') { $operationRunService->updateRun($this->operationRun, 'running'); } } $schedule = $run->schedule; if (! $schedule instanceof BackupSchedule) { $run->update([ 'status' => BackupScheduleRun::STATUS_FAILED, 'error_code' => RunErrorMapper::ERROR_UNKNOWN, 'error_message' => 'Schedule not found.', 'finished_at' => CarbonImmutable::now('UTC'), ]); if ($this->operationRun) { $this->markOperationRunFailed( run: $this->operationRun, summaryCounts: [ 'total' => 0, 'processed' => 0, 'failed' => 1, ], reasonCode: 'schedule_not_found', reason: 'Schedule not found.', ); } return; } if (! $tenant) { $run->update([ 'status' => BackupScheduleRun::STATUS_FAILED, 'error_code' => RunErrorMapper::ERROR_UNKNOWN, 'error_message' => 'Tenant not found.', 'finished_at' => CarbonImmutable::now('UTC'), ]); if ($this->operationRun) { $this->markOperationRunFailed( run: $this->operationRun, summaryCounts: [ 'total' => 0, 'processed' => 0, 'failed' => 1, ], reasonCode: 'tenant_not_found', reason: 'Tenant not found.', ); } return; } $lock = Cache::lock("backup_schedule:{$schedule->id}", 900); if (! $lock->get()) { $this->finishRun( run: $run, schedule: $schedule, status: BackupScheduleRun::STATUS_SKIPPED, errorCode: 'CONCURRENT_RUN', errorMessage: 'Another run is already in progress for this schedule.', summary: ['reason' => 'concurrent_run'], scheduleTimeService: $scheduleTimeService, ); $this->syncOperationRunFromRun( tenant: $tenant, schedule: $schedule, run: $run->refresh(), ); return; } try { $nowUtc = CarbonImmutable::now('UTC'); $run->forceFill([ 'started_at' => $run->started_at ?? $nowUtc, 'status' => BackupScheduleRun::STATUS_RUNNING, ])->save(); $this->notifyRunStarted($run, $schedule); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_started', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'backup_schedule_run_id' => $run->id, 'scheduled_for' => $run->scheduled_for?->toDateTimeString(), ], ], resourceType: 'backup_schedule_run', resourceId: (string) $run->id, status: 'success' ); $runtime = $policyTypeResolver->resolveRuntime((array) ($schedule->policy_types ?? [])); $validTypes = $runtime['valid']; $unknownTypes = $runtime['unknown']; if (empty($validTypes)) { $this->finishRun( run: $run, schedule: $schedule, status: BackupScheduleRun::STATUS_SKIPPED, errorCode: 'UNKNOWN_POLICY_TYPE', errorMessage: 'All configured policy types are unknown.', summary: [ 'unknown_policy_types' => $unknownTypes, ], scheduleTimeService: $scheduleTimeService, ); $this->syncOperationRunFromRun( tenant: $tenant, schedule: $schedule, run: $run->refresh(), ); return; } $supported = array_values(array_filter( config('tenantpilot.supported_policy_types', []), fn (array $typeConfig): bool => in_array($typeConfig['type'] ?? null, $validTypes, true), )); $syncReport = $policySyncService->syncPoliciesWithReport($tenant, $supported); $policyIds = $syncReport['synced'] ?? []; $syncFailures = $syncReport['failures'] ?? []; $backupSet = $backupService->createBackupSet( tenant: $tenant, policyIds: $policyIds, actorEmail: null, actorName: null, name: 'Scheduled backup: '.$schedule->name, includeAssignments: false, includeScopeTags: false, includeFoundations: (bool) ($schedule->include_foundations ?? false), ); $status = match ($backupSet->status) { 'completed' => BackupScheduleRun::STATUS_SUCCESS, 'partial' => BackupScheduleRun::STATUS_PARTIAL, 'failed' => BackupScheduleRun::STATUS_FAILED, default => BackupScheduleRun::STATUS_SUCCESS, }; $errorCode = null; $errorMessage = null; $summary = [ 'policies_total' => count($policyIds), 'policies_backed_up' => (int) ($backupSet->item_count ?? 0), 'sync_failures' => $syncFailures, ]; if (! empty($unknownTypes)) { $status = BackupScheduleRun::STATUS_PARTIAL; $errorCode = 'UNKNOWN_POLICY_TYPE'; $errorMessage = 'Some configured policy types are unknown and were skipped.'; $summary['unknown_policy_types'] = $unknownTypes; } $this->finishRun( run: $run, schedule: $schedule, status: $status, errorCode: $errorCode, errorMessage: $errorMessage, summary: $summary, scheduleTimeService: $scheduleTimeService, backupSetId: (string) $backupSet->id, ); $this->syncOperationRunFromRun( tenant: $tenant, schedule: $schedule, run: $run->refresh(), ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_finished', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'backup_schedule_run_id' => $run->id, 'status' => $status, 'error_code' => $errorCode, ], ], resourceType: 'backup_schedule_run', resourceId: (string) $run->id, status: in_array($status, [BackupScheduleRun::STATUS_SUCCESS], true) ? 'success' : 'partial' ); } catch (\Throwable $throwable) { $attempt = (int) method_exists($this, 'attempts') ? $this->attempts() : 1; $mapped = $errorMapper->map($throwable, $attempt, $this->tries); if ($mapped['shouldRetry']) { if ($this->operationRun) { /** @var OperationRunService $operationRunService */ $operationRunService = app(OperationRunService::class); $operationRunService->updateRun($this->operationRun, 'running', 'pending'); } $this->release($mapped['delay']); return; } $this->finishRun( run: $run, schedule: $schedule, status: BackupScheduleRun::STATUS_FAILED, errorCode: $mapped['error_code'], errorMessage: $mapped['error_message'], summary: [ 'exception' => get_class($throwable), 'attempt' => $attempt, ], scheduleTimeService: $scheduleTimeService, ); $this->syncOperationRunFromRun( tenant: $tenant, schedule: $schedule, run: $run->refresh(), ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_failed', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'backup_schedule_run_id' => $run->id, 'error_code' => $mapped['error_code'], ], ], resourceType: 'backup_schedule_run', resourceId: (string) $run->id, status: 'failed' ); } finally { optional($lock)->release(); } } private function notifyRunStarted(BackupScheduleRun $run, BackupSchedule $schedule): void { $user = $run->user; if (! $user) { return; } $notification = Notification::make() ->title('Backup started') ->body(sprintf('Schedule "%s" has started.', $schedule->name)) ->info() ->actions([ Action::make('view_run') ->label('View run') ->url($this->operationRun ? OperationRunLinks::view($this->operationRun, $run->tenant) : OperationRunLinks::index($run->tenant)), ]); $notification->sendToDatabase($user); } private function notifyRunFinished(BackupScheduleRun $run, BackupSchedule $schedule): void { $user = $run->user; if (! $user) { return; } $title = match ($run->status) { BackupScheduleRun::STATUS_SUCCESS => 'Backup completed', BackupScheduleRun::STATUS_PARTIAL => 'Backup completed (partial)', BackupScheduleRun::STATUS_SKIPPED => 'Backup skipped', default => 'Backup failed', }; $notification = Notification::make() ->title($title) ->body(sprintf('Schedule "%s" finished with status: %s.', $schedule->name, $run->status)); if (filled($run->error_message)) { $notification->body($notification->getBody()."\n".$run->error_message); } match ($run->status) { BackupScheduleRun::STATUS_SUCCESS => $notification->success(), BackupScheduleRun::STATUS_PARTIAL, BackupScheduleRun::STATUS_SKIPPED => $notification->warning(), default => $notification->danger(), }; $notification ->actions([ Action::make('view_run') ->label('View run') ->url($this->operationRun ? OperationRunLinks::view($this->operationRun, $run->tenant) : OperationRunLinks::index($run->tenant)), ]) ->sendToDatabase($user); } private function syncOperationRunFromRun( Tenant $tenant, BackupSchedule $schedule, BackupScheduleRun $run, ): void { if (! $this->operationRun) { return; } $outcome = match ($run->status) { BackupScheduleRun::STATUS_SUCCESS => 'succeeded', BackupScheduleRun::STATUS_PARTIAL => 'partially_succeeded', // Note: 'cancelled' is a reserved OperationRun outcome token. // We treat schedule SKIPPED/CANCELED as terminal failures with a failure entry. BackupScheduleRun::STATUS_SKIPPED, BackupScheduleRun::STATUS_CANCELED => 'failed', default => 'failed', }; $summary = is_array($run->summary) ? $run->summary : []; $syncFailures = $summary['sync_failures'] ?? []; $policiesTotal = (int) ($summary['policies_total'] ?? 0); $policiesBackedUp = (int) ($summary['policies_backed_up'] ?? 0); $syncFailureCount = is_array($syncFailures) ? count($syncFailures) : 0; $failedCount = max(0, $policiesTotal - $policiesBackedUp); $summaryCounts = [ 'total' => $policiesTotal, 'processed' => $policiesTotal, 'succeeded' => $policiesBackedUp, 'failed' => $failedCount, 'skipped' => 0, 'created' => filled($run->backup_set_id) ? 1 : 0, 'updated' => $policiesBackedUp, 'items' => $policiesTotal, ]; $failures = []; if (filled($run->error_message) || filled($run->error_code)) { $failures[] = [ 'code' => strtolower((string) ($run->error_code ?: 'backup_schedule_error')), 'message' => (string) ($run->error_message ?: 'Backup schedule run failed.'), ]; } if (is_array($syncFailures)) { foreach ($syncFailures as $failure) { if (! is_array($failure)) { continue; } $policyType = (string) ($failure['policy_type'] ?? 'unknown'); $status = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null; $errors = $failure['errors'] ?? null; $firstErrorMessage = null; if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) { $firstErrorMessage = $errors[0]['message'] ?? null; } $message = $status !== null ? "{$policyType}: Graph returned {$status}" : "{$policyType}: Graph request failed"; if (is_string($firstErrorMessage) && $firstErrorMessage !== '') { $message .= ' - '.trim($firstErrorMessage); } $failures[] = [ 'code' => $status !== null ? 'graph_http_'.(string) $status : 'graph_error', 'message' => $message, ]; } } /** @var OperationRunService $operationRunService */ $operationRunService = app(OperationRunService::class); $this->operationRun->update([ 'context' => array_merge($this->operationRun->context ?? [], [ 'backup_schedule_id' => (int) $schedule->getKey(), 'backup_schedule_run_id' => (int) $run->getKey(), 'backup_set_id' => $run->backup_set_id ? (int) $run->backup_set_id : null, ]), ]); $operationRunService->updateRun( $this->operationRun, status: 'completed', outcome: $outcome, summaryCounts: $summaryCounts, failures: $failures, ); } private function markOperationRunFailed( OperationRun $run, array $summaryCounts, string $reasonCode, string $reason, ): void { /** @var OperationRunService $operationRunService */ $operationRunService = app(OperationRunService::class); $operationRunService->updateRun( $run, status: 'completed', outcome: 'failed', summaryCounts: $summaryCounts, failures: [ [ 'code' => $reasonCode, 'message' => $reason, ], ], ); } private function resolveOperationRunFromContext(Tenant $tenant, BackupScheduleRun $run): void { if ($this->operationRun) { return; } $operationRun = OperationRun::query() ->where('tenant_id', $tenant->getKey()) ->whereIn('type', ['backup_schedule.run_now', 'backup_schedule.retry']) ->whereIn('status', ['queued', 'running']) ->where('context->backup_schedule_run_id', (int) $run->getKey()) ->latest('id') ->first(); if ($operationRun instanceof OperationRun) { $this->operationRun = $operationRun; } } private function finishRun( BackupScheduleRun $run, BackupSchedule $schedule, string $status, ?string $errorCode, ?string $errorMessage, array $summary, ScheduleTimeService $scheduleTimeService, ?string $backupSetId = null, ): void { $nowUtc = CarbonImmutable::now('UTC'); $run->forceFill([ 'status' => $status, 'error_code' => $errorCode, 'error_message' => $errorMessage, 'summary' => Arr::wrap($summary), 'finished_at' => $nowUtc, 'backup_set_id' => $backupSetId, ])->save(); $schedule->forceFill([ 'last_run_at' => $nowUtc, 'last_run_status' => $status, 'next_run_at' => $scheduleTimeService->nextRunFor($schedule, $nowUtc), ])->saveQuietly(); $this->notifyRunFinished($run, $schedule); if ($backupSetId && in_array($status, [BackupScheduleRun::STATUS_SUCCESS, BackupScheduleRun::STATUS_PARTIAL], true)) { Bus::dispatch(new ApplyBackupScheduleRetentionJob($schedule->id)); } } }