operationRun = $operationRun; } public function middleware(): array { return [new TrackOperationRun]; } public function handle( PolicySyncService $policySyncService, BackupService $backupService, PolicyTypeResolver $policyTypeResolver, ScheduleTimeService $scheduleTimeService, AuditLogger $auditLogger, RunErrorMapper $errorMapper, ): void { if (! $this->operationRun) { $this->fail(new \RuntimeException('OperationRun context is required for RunBackupScheduleJob.')); return; } $backupScheduleId = $this->resolveBackupScheduleId(); if ($backupScheduleId <= 0) { $this->markOperationRunFailed( run: $this->operationRun, summaryCounts: [], reasonCode: 'schedule_not_provided', reason: 'No backup schedule was provided for this run.', ); return; } $schedule = BackupSchedule::query() ->withTrashed() ->with('tenant') ->find($backupScheduleId); if (! $schedule instanceof BackupSchedule) { $this->markOperationRunFailed( run: $this->operationRun, summaryCounts: [], reasonCode: 'schedule_not_found', reason: 'Schedule not found.', ); return; } $tenant = $schedule->tenant; if (! $tenant instanceof Tenant) { $this->markOperationRunFailed( run: $this->operationRun, summaryCounts: [], reasonCode: 'tenant_not_found', reason: 'Tenant not found.', ); return; } /** @var OperationRunService $operationRunService */ $operationRunService = app(OperationRunService::class); $this->operationRun->update([ 'context' => array_merge($this->operationRun->context ?? [], [ 'backup_schedule_id' => (int) $schedule->getKey(), ]), ]); if ($this->operationRun->status === 'queued') { $operationRunService->updateRun($this->operationRun, 'running'); } if ($schedule->trashed()) { $nowUtc = CarbonImmutable::now('UTC'); $this->finishSchedule( schedule: $schedule, status: self::STATUS_SKIPPED, scheduleTimeService: $scheduleTimeService, nowUtc: $nowUtc, ); $operationRunService->updateRun( $this->operationRun, status: 'completed', outcome: OperationRunOutcome::Blocked->value, summaryCounts: [ 'total' => 0, 'processed' => 0, 'failed' => 0, 'skipped' => 1, ], failures: [ [ 'code' => 'schedule_archived', 'message' => 'Schedule is archived; run will not execute.', ], ], ); $this->notifyScheduleRunFinished( tenant: $tenant, schedule: $schedule, status: self::STATUS_SKIPPED, errorMessage: 'Schedule is archived; run will not execute.', ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_skipped', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'operation_run_id' => $this->operationRun->getKey(), 'reason' => 'schedule_archived', ], ], resourceType: 'operation_run', resourceId: (string) $this->operationRun->getKey(), status: 'partial' ); return; } $lock = Cache::lock("backup_schedule:{$schedule->id}", 900); if (! $lock->get()) { $nowUtc = CarbonImmutable::now('UTC'); $this->finishSchedule( schedule: $schedule, status: self::STATUS_SKIPPED, scheduleTimeService: $scheduleTimeService, nowUtc: $nowUtc, ); $operationRunService->updateRun( $this->operationRun, status: 'completed', outcome: OperationRunOutcome::Blocked->value, summaryCounts: [ 'total' => 0, 'processed' => 0, 'failed' => 0, 'skipped' => 1, ], failures: [ [ 'code' => 'concurrent_run', 'message' => 'Another run is already in progress for this schedule.', ], ], ); $this->notifyScheduleRunFinished( tenant: $tenant, schedule: $schedule, status: self::STATUS_SKIPPED, errorMessage: 'Another run is already in progress for this schedule.', ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_skipped', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'operation_run_id' => $this->operationRun->getKey(), 'reason' => 'concurrent_run', ], ], resourceType: 'operation_run', resourceId: (string) $this->operationRun->getKey(), status: 'partial' ); return; } try { $nowUtc = CarbonImmutable::now('UTC'); $this->notifyScheduleRunStarted(tenant: $tenant, schedule: $schedule); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_started', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'operation_run_id' => $this->operationRun->getKey(), ], ], resourceType: 'operation_run', resourceId: (string) $this->operationRun->getKey(), status: 'success' ); $runtime = $policyTypeResolver->resolveRuntime((array) ($schedule->policy_types ?? [])); $validTypes = $runtime['valid']; $unknownTypes = $runtime['unknown']; if (empty($validTypes)) { $this->finishSchedule( schedule: $schedule, status: self::STATUS_SKIPPED, scheduleTimeService: $scheduleTimeService, nowUtc: $nowUtc, ); $operationRunService->updateRun( $this->operationRun, status: 'completed', outcome: OperationRunOutcome::Blocked->value, summaryCounts: [ 'total' => 0, 'processed' => 0, 'failed' => 0, 'skipped' => 1, ], failures: [ [ 'code' => 'unknown_policy_type', 'message' => 'All configured policy types are unknown.', ], ], ); $this->notifyScheduleRunFinished( tenant: $tenant, schedule: $schedule, status: self::STATUS_SKIPPED, errorMessage: 'All configured policy types are unknown.', ); return; } $supported = array_values(array_filter( config('tenantpilot.supported_policy_types', []), fn (array $typeConfig): bool => in_array($typeConfig['type'] ?? null, $validTypes, true), )); $syncReport = $policySyncService->syncPoliciesWithReport($tenant, $supported); $policyIds = $syncReport['synced'] ?? []; $syncFailures = $syncReport['failures'] ?? []; $backupSet = $backupService->createBackupSet( tenant: $tenant, policyIds: $policyIds, actorEmail: null, actorName: null, name: 'Scheduled backup: '.$schedule->name, includeAssignments: false, includeScopeTags: false, includeFoundations: (bool) ($schedule->include_foundations ?? false), ); $status = match ($backupSet->status) { 'completed' => self::STATUS_SUCCESS, 'partial' => self::STATUS_PARTIAL, 'failed' => self::STATUS_FAILED, default => self::STATUS_SUCCESS, }; $errorCode = null; $errorMessage = null; if (! empty($unknownTypes)) { $status = self::STATUS_PARTIAL; $errorCode = 'UNKNOWN_POLICY_TYPE'; $errorMessage = 'Some configured policy types are unknown and were skipped.'; } $policiesTotal = count($policyIds); $policiesBackedUp = (int) ($backupSet->item_count ?? 0); $failedCount = max(0, $policiesTotal - $policiesBackedUp); $summaryCounts = [ 'total' => $policiesTotal, 'processed' => $policiesTotal, 'succeeded' => $policiesBackedUp, 'failed' => $failedCount, 'skipped' => 0, 'created' => 1, 'updated' => $policiesBackedUp, 'items' => $policiesTotal, ]; $failures = []; if (is_string($errorMessage) && $errorMessage !== '') { $failures[] = [ 'code' => strtolower((string) ($errorCode ?: 'backup_schedule_error')), 'message' => $errorMessage, ]; } if (is_array($syncFailures)) { foreach ($syncFailures as $failure) { if (! is_array($failure)) { continue; } $policyType = (string) ($failure['policy_type'] ?? 'unknown'); $httpStatus = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null; $errors = $failure['errors'] ?? null; $firstErrorMessage = null; if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) { $firstErrorMessage = $errors[0]['message'] ?? null; } $message = $httpStatus !== null ? "{$policyType}: Graph returned {$httpStatus}" : "{$policyType}: Graph request failed"; if (is_string($firstErrorMessage) && $firstErrorMessage !== '') { $message .= ' - '.trim($firstErrorMessage); } $failures[] = [ 'code' => $httpStatus !== null ? 'graph_http_'.(string) $httpStatus : 'graph_error', 'message' => $message, ]; } } $this->operationRun->update([ 'context' => array_merge($this->operationRun->context ?? [], [ 'backup_schedule_id' => (int) $schedule->getKey(), 'backup_set_id' => (int) $backupSet->getKey(), ]), ]); $outcome = match ($status) { self::STATUS_SUCCESS => OperationRunOutcome::Succeeded->value, self::STATUS_PARTIAL => OperationRunOutcome::PartiallySucceeded->value, self::STATUS_SKIPPED => OperationRunOutcome::Blocked->value, default => OperationRunOutcome::Failed->value, }; $operationRunService->updateRun( $this->operationRun, status: 'completed', outcome: $outcome, summaryCounts: $summaryCounts, failures: $failures, ); $this->finishSchedule( schedule: $schedule, status: $status, scheduleTimeService: $scheduleTimeService, nowUtc: $nowUtc, ); $this->notifyScheduleRunFinished( tenant: $tenant, schedule: $schedule, status: $status, errorMessage: $errorMessage, ); if (in_array($status, [self::STATUS_SUCCESS, self::STATUS_PARTIAL], true)) { Bus::dispatch(new ApplyBackupScheduleRetentionJob((int) $schedule->getKey())); } $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_finished', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'operation_run_id' => $this->operationRun->getKey(), 'status' => $status, 'error_code' => $errorCode, ], ], resourceType: 'operation_run', resourceId: (string) $this->operationRun->getKey(), status: in_array($status, [self::STATUS_SUCCESS], true) ? 'success' : 'partial' ); } catch (\Throwable $throwable) { $attempt = (int) method_exists($this, 'attempts') ? $this->attempts() : 1; $mapped = $errorMapper->map($throwable, $attempt, $this->tries); if ($mapped['shouldRetry']) { $operationRunService->updateRun($this->operationRun, 'running', OperationRunOutcome::Pending->value); $this->release($mapped['delay']); return; } $nowUtc = CarbonImmutable::now('UTC'); $this->finishSchedule( schedule: $schedule, status: self::STATUS_FAILED, scheduleTimeService: $scheduleTimeService, nowUtc: $nowUtc, ); $operationRunService->updateRun( $this->operationRun, status: 'completed', outcome: OperationRunOutcome::Failed->value, summaryCounts: [ 'total' => 0, 'processed' => 0, 'failed' => 1, ], failures: [ [ 'code' => strtolower((string) $mapped['error_code']), 'message' => (string) $mapped['error_message'], ], ], ); $this->notifyScheduleRunFinished( tenant: $tenant, schedule: $schedule, status: self::STATUS_FAILED, errorMessage: (string) $mapped['error_message'], ); $auditLogger->log( tenant: $tenant, action: 'backup_schedule.run_failed', context: [ 'metadata' => [ 'backup_schedule_id' => $schedule->id, 'operation_run_id' => $this->operationRun->getKey(), 'error_code' => $mapped['error_code'], ], ], resourceType: 'operation_run', resourceId: (string) $this->operationRun->getKey(), status: 'failed' ); } finally { optional($lock)->release(); } } private function resolveBackupScheduleId(): int { if ($this->backupScheduleId !== null && $this->backupScheduleId > 0) { return $this->backupScheduleId; } $contextScheduleId = data_get($this->operationRun?->context, 'backup_schedule_id'); return is_numeric($contextScheduleId) ? (int) $contextScheduleId : 0; } private function notifyScheduleRunStarted(Tenant $tenant, BackupSchedule $schedule): void { $userId = $this->operationRun?->user_id; if (! $userId) { return; } $user = User::query()->find($userId); if (! $user instanceof User) { return; } Notification::make() ->title('Backup started') ->body(sprintf('Schedule "%s" has started.', $schedule->name)) ->info() ->actions([ Action::make('view_run') ->label('View run') ->url(OperationRunLinks::view($this->operationRun, $tenant)), ]) ->sendToDatabase($user); } private function notifyScheduleRunFinished( Tenant $tenant, BackupSchedule $schedule, string $status, ?string $errorMessage, ): void { $userId = $this->operationRun?->user_id; if (! $userId) { return; } $user = User::query()->find($userId); if (! $user instanceof User) { return; } $title = match ($status) { self::STATUS_SUCCESS => 'Backup completed', self::STATUS_PARTIAL => 'Backup completed (partial)', self::STATUS_SKIPPED => 'Backup skipped', default => 'Backup failed', }; $notification = Notification::make() ->title($title) ->body(sprintf('Schedule "%s" finished with status: %s.', $schedule->name, $status)); if (is_string($errorMessage) && $errorMessage !== '') { $notification->body($notification->getBody()."\n".$errorMessage); } match ($status) { self::STATUS_SUCCESS => $notification->success(), self::STATUS_PARTIAL, self::STATUS_SKIPPED => $notification->warning(), default => $notification->danger(), }; $notification ->actions([ Action::make('view_run') ->label('View run') ->url(OperationRunLinks::view($this->operationRun, $tenant)), ]) ->sendToDatabase($user); } private function finishSchedule( BackupSchedule $schedule, string $status, ScheduleTimeService $scheduleTimeService, CarbonImmutable $nowUtc, ): void { $schedule->forceFill([ 'last_run_at' => $nowUtc, 'last_run_status' => $status, 'next_run_at' => $scheduleTimeService->nextRunFor($schedule, $nowUtc), ])->saveQuietly(); } private function markOperationRunFailed( OperationRun $run, array $summaryCounts, string $reasonCode, string $reason, ): void { /** @var OperationRunService $operationRunService */ $operationRunService = app(OperationRunService::class); $operationRunService->updateRun( $run, status: 'completed', outcome: OperationRunOutcome::Failed->value, summaryCounts: $summaryCounts, failures: [ [ 'code' => $reasonCode, 'message' => $reason, ], ], ); } }