with(['schedule', 'tenant', 'user']) ->find($this->backupScheduleRunId); if (! $run) { return; } $bulkRun = $this->bulkRunId ? BulkOperationRun::query()->with(['tenant', 'user'])->find($this->bulkRunId) : null; if ( $bulkRun && ($bulkRun->tenant_id !== $run->tenant_id || $bulkRun->user_id !== $run->user_id) ) { $bulkRun = null; } if ($bulkRun && $bulkRun->status === 'pending') { $bulkOperationService->start($bulkRun); } $schedule = $run->schedule; if (! $schedule instanceof BackupSchedule) { $run->update([ 'status' => BackupScheduleRun::STATUS_FAILED, 'error_code' => RunErrorMapper::ERROR_UNKNOWN, 'error_message' => 'Schedule not found.', 'finished_at' => CarbonImmutable::now('UTC'), ]); return; } $tenant = $run->tenant; if (! $tenant) { $run->update([ 'status' => BackupScheduleRun::STATUS_FAILED, 'error_code' => RunErrorMapper::ERROR_UNKNOWN, 'error_message' => 'Tenant not found.', 'finished_at' => CarbonImmutable::now('UTC'), ]); return; } $lock = Cache::lock("backup_schedule:{$schedule->id}", 900); if (! $lock->get()) { $this->finishRun( run: $run, schedule: $schedule, status: BackupScheduleRun::STATUS_SKIPPED, errorCode: 'CONCURRENT_RUN', errorMessage: 'Another run is already in progress for this schedule.', summary: ['reason' => 'concurrent_run'], scheduleTimeService: $scheduleTimeService, bulkRunId: $this->bulkRunId, ); return; } try { $nowUtc = CarbonImmutable::now('UTC'); $run->forceFill([ 'started_at' => $run->started_at ?? $nowUtc, 'status' => BackupScheduleRun::STATUS_RUNNING, ])->save(); $this->notifyRunStarted($run, $schedule); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_started', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'backup_schedule_run_id' => $run->id, 'scheduled_for' => $run->scheduled_for?->toDateTimeString(), ], ], resourceType: 'backup_schedule_run', resourceId: (string) $run->id, status: 'success' ); $runtime = $policyTypeResolver->resolveRuntime((array) ($schedule->policy_types ?? [])); $validTypes = $runtime['valid']; $unknownTypes = $runtime['unknown']; if (empty($validTypes)) { $this->finishRun( run: $run, schedule: $schedule, status: BackupScheduleRun::STATUS_SKIPPED, errorCode: 'UNKNOWN_POLICY_TYPE', errorMessage: 'All configured policy types are unknown.', summary: [ 'unknown_policy_types' => $unknownTypes, ], scheduleTimeService: $scheduleTimeService, bulkRunId: $this->bulkRunId, ); return; } $supported = array_values(array_filter( config('tenantpilot.supported_policy_types', []), fn (array $typeConfig): bool => in_array($typeConfig['type'] ?? null, $validTypes, true), )); $syncReport = $policySyncService->syncPoliciesWithReport($tenant, $supported); $policyIds = $syncReport['synced'] ?? []; $syncFailures = $syncReport['failures'] ?? []; $backupSet = $backupService->createBackupSet( tenant: $tenant, policyIds: $policyIds, actorEmail: null, actorName: null, name: 'Scheduled backup: '.$schedule->name, includeAssignments: false, includeScopeTags: false, includeFoundations: (bool) ($schedule->include_foundations ?? false), ); $status = match ($backupSet->status) { 'completed' => BackupScheduleRun::STATUS_SUCCESS, 'partial' => BackupScheduleRun::STATUS_PARTIAL, 'failed' => BackupScheduleRun::STATUS_FAILED, default => BackupScheduleRun::STATUS_SUCCESS, }; $errorCode = null; $errorMessage = null; $summary = [ 'policies_total' => count($policyIds), 'policies_backed_up' => (int) ($backupSet->item_count ?? 0), 'sync_failures' => $syncFailures, ]; if (! empty($unknownTypes)) { $status = BackupScheduleRun::STATUS_PARTIAL; $errorCode = 'UNKNOWN_POLICY_TYPE'; $errorMessage = 'Some configured policy types are unknown and were skipped.'; $summary['unknown_policy_types'] = $unknownTypes; } $this->finishRun( run: $run, schedule: $schedule, status: $status, errorCode: $errorCode, errorMessage: $errorMessage, summary: $summary, scheduleTimeService: $scheduleTimeService, backupSetId: (string) $backupSet->id, bulkRunId: $this->bulkRunId, ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_finished', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'backup_schedule_run_id' => $run->id, 'status' => $status, 'error_code' => $errorCode, ], ], resourceType: 'backup_schedule_run', resourceId: (string) $run->id, status: in_array($status, [BackupScheduleRun::STATUS_SUCCESS], true) ? 'success' : 'partial' ); } catch (\Throwable $throwable) { $attempt = (int) method_exists($this, 'attempts') ? $this->attempts() : 1; $mapped = $errorMapper->map($throwable, $attempt, $this->tries); if ($mapped['shouldRetry']) { $this->release($mapped['delay']); return; } $this->finishRun( run: $run, schedule: $schedule, status: BackupScheduleRun::STATUS_FAILED, errorCode: $mapped['error_code'], errorMessage: $mapped['error_message'], summary: [ 'exception' => get_class($throwable), 'attempt' => $attempt, ], scheduleTimeService: $scheduleTimeService, bulkRunId: $this->bulkRunId, ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_failed', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'backup_schedule_run_id' => $run->id, 'error_code' => $mapped['error_code'], ], ], resourceType: 'backup_schedule_run', resourceId: (string) $run->id, status: 'failed' ); } finally { optional($lock)->release(); } } private function notifyRunStarted(BackupScheduleRun $run, BackupSchedule $schedule): void { $user = $run->user; if (! $user) { return; } $notification = Notification::make() ->title('Backup started') ->body(sprintf('Schedule "%s" has started.', $schedule->name)) ->info(); $user->notifyNow($notification->toDatabase()); DatabaseNotificationsSent::dispatch($user); } private function notifyRunFinished(BackupScheduleRun $run, BackupSchedule $schedule): void { $user = $run->user; if (! $user) { return; } $title = match ($run->status) { BackupScheduleRun::STATUS_SUCCESS => 'Backup completed', BackupScheduleRun::STATUS_PARTIAL => 'Backup completed (partial)', BackupScheduleRun::STATUS_SKIPPED => 'Backup skipped', default => 'Backup failed', }; $notification = Notification::make() ->title($title) ->body(sprintf('Schedule "%s" finished with status: %s.', $schedule->name, $run->status)); if (filled($run->error_message)) { $notification->body($notification->getBody()."\n".$run->error_message); } match ($run->status) { BackupScheduleRun::STATUS_SUCCESS => $notification->success(), BackupScheduleRun::STATUS_PARTIAL, BackupScheduleRun::STATUS_SKIPPED => $notification->warning(), default => $notification->danger(), }; $user->notifyNow($notification->toDatabase()); DatabaseNotificationsSent::dispatch($user); } private function finishRun( BackupScheduleRun $run, BackupSchedule $schedule, string $status, ?string $errorCode, ?string $errorMessage, array $summary, ScheduleTimeService $scheduleTimeService, ?string $backupSetId = null, ?int $bulkRunId = null, ): void { $nowUtc = CarbonImmutable::now('UTC'); $run->forceFill([ 'status' => $status, 'error_code' => $errorCode, 'error_message' => $errorMessage, 'summary' => Arr::wrap($summary), 'finished_at' => $nowUtc, 'backup_set_id' => $backupSetId, ])->save(); $schedule->forceFill([ 'last_run_at' => $nowUtc, 'last_run_status' => $status, 'next_run_at' => $scheduleTimeService->nextRunFor($schedule, $nowUtc), ])->saveQuietly(); $this->notifyRunFinished($run, $schedule); if ($bulkRunId) { $bulkRun = BulkOperationRun::query()->with(['tenant', 'user'])->find($bulkRunId); if ( $bulkRun && ($bulkRun->tenant_id === $run->tenant_id) && ($bulkRun->user_id === $run->user_id) && in_array($bulkRun->status, ['pending', 'running'], true) ) { $service = app(BulkOperationService::class); $itemId = (string) $run->backup_schedule_id; match ($status) { BackupScheduleRun::STATUS_SUCCESS => $service->recordSuccess($bulkRun), BackupScheduleRun::STATUS_SKIPPED => $service->recordSkippedWithReason( $bulkRun, $itemId, $errorMessage ?: 'Skipped', ), BackupScheduleRun::STATUS_PARTIAL => $service->recordFailure( $bulkRun, $itemId, $errorMessage ?: 'Completed partially', ), default => $service->recordFailure( $bulkRun, $itemId, $errorMessage ?: ($errorCode ?: 'Failed'), ), }; $bulkRun->refresh(); if ( in_array($bulkRun->status, ['pending', 'running'], true) && $bulkRun->processed_items >= $bulkRun->total_items ) { $service->complete($bulkRun); } } } if ($backupSetId && in_array($status, [BackupScheduleRun::STATUS_SUCCESS, BackupScheduleRun::STATUS_PARTIAL], true)) { Bus::dispatch(new ApplyBackupScheduleRetentionJob($schedule->id)); } } }