option('tenant'))); $olderThanMinutes = max(0, (int) $this->option('older-than')); $dryRun = (bool) $this->option('dry-run'); $query = OperationRun::query() ->whereIn('type', ['backup_schedule.run_now', 'backup_schedule.retry']) ->whereIn('status', ['queued', 'running']); if ($olderThanMinutes > 0) { $query->where('created_at', '<', now()->subMinutes($olderThanMinutes)); } if ($tenantIdentifiers !== []) { $tenantIds = $this->resolveTenantIds($tenantIdentifiers); if ($tenantIds === []) { $this->info('No tenants matched the provided identifiers.'); return self::SUCCESS; } $query->whereIn('tenant_id', $tenantIds); } $reconciled = 0; $skipped = 0; $failed = 0; foreach ($query->cursor() as $operationRun) { $backupScheduleRunId = data_get($operationRun->context, 'backup_schedule_run_id'); if (! is_numeric($backupScheduleRunId)) { $skipped++; continue; } $scheduleRun = BackupScheduleRun::query() ->whereKey((int) $backupScheduleRunId) ->where('tenant_id', $operationRun->tenant_id) ->first(); if (! $scheduleRun) { if (! $dryRun) { $operationRunService->updateRun( $operationRun, status: 'completed', outcome: 'failed', failures: [ [ 'code' => 'backup_schedule_run.not_found', 'message' => RunFailureSanitizer::sanitizeMessage('Backup schedule run not found.'), ], ], ); } $failed++; continue; } if ($scheduleRun->status === BackupScheduleRun::STATUS_RUNNING) { if (! $dryRun) { $operationRunService->updateRun($operationRun, 'running', 'pending'); if ($scheduleRun->started_at) { $operationRun->forceFill(['started_at' => $scheduleRun->started_at])->save(); } } $reconciled++; continue; } $outcome = match ($scheduleRun->status) { BackupScheduleRun::STATUS_SUCCESS => 'succeeded', BackupScheduleRun::STATUS_PARTIAL => 'partially_succeeded', BackupScheduleRun::STATUS_SKIPPED => 'succeeded', BackupScheduleRun::STATUS_CANCELED => 'failed', default => 'failed', }; $summary = is_array($scheduleRun->summary) ? $scheduleRun->summary : []; $syncFailures = $summary['sync_failures'] ?? []; $policiesTotal = (int) ($summary['policies_total'] ?? 0); $policiesBackedUp = (int) ($summary['policies_backed_up'] ?? 0); $syncFailuresCount = is_array($syncFailures) ? count($syncFailures) : 0; $processed = $policiesBackedUp + $syncFailuresCount; if ($policiesTotal > 0) { $processed = min($policiesTotal, $processed); } $summaryCounts = array_filter([ 'total' => $policiesTotal, 'processed' => $processed, 'succeeded' => $policiesBackedUp, 'failed' => $syncFailuresCount, 'skipped' => $scheduleRun->status === BackupScheduleRun::STATUS_SKIPPED ? 1 : 0, 'items' => $policiesTotal, ], fn (mixed $value): bool => is_int($value) && $value !== 0); $failures = []; if ($scheduleRun->status === BackupScheduleRun::STATUS_CANCELED) { $failures[] = [ 'code' => 'backup_schedule_run.cancelled', 'message' => 'Backup schedule run was cancelled.', ]; } if (filled($scheduleRun->error_message) || filled($scheduleRun->error_code)) { $failures[] = [ 'code' => (string) ($scheduleRun->error_code ?: 'backup_schedule_run.error'), 'message' => RunFailureSanitizer::sanitizeMessage((string) ($scheduleRun->error_message ?: 'Backup schedule run failed.')), ]; } if (is_array($syncFailures)) { foreach ($syncFailures as $failure) { if (! is_array($failure)) { continue; } $policyType = (string) ($failure['policy_type'] ?? 'unknown'); $status = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null; $errors = $failure['errors'] ?? null; $firstErrorMessage = null; if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) { $firstErrorMessage = $errors[0]['message'] ?? null; } $message = $status !== null ? "{$policyType}: Graph returned {$status}" : "{$policyType}: Graph request failed"; if (is_string($firstErrorMessage) && $firstErrorMessage !== '') { $message .= ' - '.trim($firstErrorMessage); } $failures[] = [ 'code' => $status !== null ? "graph.http_{$status}" : 'graph.error', 'message' => RunFailureSanitizer::sanitizeMessage($message), ]; } } if (! $dryRun) { $operationRun->update([ 'context' => array_merge($operationRun->context ?? [], [ 'backup_schedule_id' => (int) $scheduleRun->backup_schedule_id, 'backup_schedule_run_id' => (int) $scheduleRun->getKey(), ]), ]); $operationRunService->updateRun( $operationRun, status: 'completed', outcome: $outcome, summaryCounts: $summaryCounts, failures: $failures, ); $operationRun->forceFill([ 'started_at' => $scheduleRun->started_at ?? $operationRun->started_at, 'completed_at' => $scheduleRun->finished_at ?? $operationRun->completed_at, ])->save(); } $reconciled++; } $this->info(sprintf( 'Reconciled %d run(s), skipped %d, failed %d.', $reconciled, $skipped, $failed, )); if ($dryRun) { $this->comment('Dry-run: no changes written.'); } return self::SUCCESS; } /** * @param array $tenantIdentifiers * @return array */ private function resolveTenantIds(array $tenantIdentifiers): array { $tenantIds = []; foreach ($tenantIdentifiers as $identifier) { $tenant = Tenant::query() ->forTenant($identifier) ->first(); if ($tenant) { $tenantIds[] = (int) $tenant->getKey(); } } return array_values(array_unique($tenantIds)); } }