option('tenant'))); $olderThanMinutes = max(0, (int) $this->option('older-than')); $dryRun = (bool) $this->option('dry-run'); $query = OperationRun::query() ->whereIn('type', ['backup_schedule.run_now', 'backup_schedule.retry']) ->whereIn('status', ['queued', 'running']); if ($olderThanMinutes > 0) { $query->where('created_at', '<', now()->subMinutes($olderThanMinutes)); } if ($tenantIdentifiers !== []) { $tenantIds = $this->resolveTenantIds($tenantIdentifiers); if ($tenantIds === []) { $this->info('No tenants matched the provided identifiers.'); return self::SUCCESS; } $query->whereIn('tenant_id', $tenantIds); } $reconciled = 0; $skipped = 0; $failed = 0; foreach ($query->cursor() as $operationRun) { $backupScheduleRunId = data_get($operationRun->context, 'backup_schedule_run_id'); if (! is_numeric($backupScheduleRunId)) { $skipped++; continue; } $scheduleRun = BackupScheduleRun::query() ->whereKey((int) $backupScheduleRunId) ->where('tenant_id', $operationRun->tenant_id) ->first(); if (! $scheduleRun) { if (! $dryRun) { $operationRunService->updateRun( $operationRun, status: 'completed', outcome: 'failed', failures: [ [ 'code' => 'RUN_NOT_FOUND', 'message' => $bulkOperationService->sanitizeFailureReason('Backup schedule run not found.'), ], ], ); } $failed++; continue; } if ($scheduleRun->status === BackupScheduleRun::STATUS_RUNNING) { if (! $dryRun) { $operationRunService->updateRun($operationRun, 'running', 'pending'); if ($scheduleRun->started_at) { $operationRun->forceFill(['started_at' => $scheduleRun->started_at])->save(); } } $reconciled++; continue; } $outcome = match ($scheduleRun->status) { BackupScheduleRun::STATUS_SUCCESS => 'succeeded', BackupScheduleRun::STATUS_PARTIAL => 'partially_succeeded', BackupScheduleRun::STATUS_SKIPPED, BackupScheduleRun::STATUS_CANCELED => 'cancelled', default => 'failed', }; $summary = is_array($scheduleRun->summary) ? $scheduleRun->summary : []; $syncFailures = $summary['sync_failures'] ?? []; $summaryCounts = [ 'backup_schedule_id' => (int) $scheduleRun->backup_schedule_id, 'backup_schedule_run_id' => (int) $scheduleRun->getKey(), 'backup_set_id' => $scheduleRun->backup_set_id ? (int) $scheduleRun->backup_set_id : null, 'policies_total' => (int) ($summary['policies_total'] ?? 0), 'policies_backed_up' => (int) ($summary['policies_backed_up'] ?? 0), 'sync_failures' => is_array($syncFailures) ? count($syncFailures) : 0, ]; $summaryCounts = array_filter($summaryCounts, fn (mixed $value): bool => $value !== null); $failures = []; if (filled($scheduleRun->error_message) || filled($scheduleRun->error_code)) { $failures[] = [ 'code' => (string) ($scheduleRun->error_code ?: 'BACKUP_SCHEDULE_ERROR'), 'message' => $bulkOperationService->sanitizeFailureReason((string) ($scheduleRun->error_message ?: 'Backup schedule run failed.')), ]; } if (is_array($syncFailures)) { foreach ($syncFailures as $failure) { if (! is_array($failure)) { continue; } $policyType = (string) ($failure['policy_type'] ?? 'unknown'); $status = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null; $errors = $failure['errors'] ?? null; $firstErrorMessage = null; if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) { $firstErrorMessage = $errors[0]['message'] ?? null; } $message = $status !== null ? "{$policyType}: Graph returned {$status}" : "{$policyType}: Graph request failed"; if (is_string($firstErrorMessage) && $firstErrorMessage !== '') { $message .= ' - '.trim($firstErrorMessage); } $failures[] = [ 'code' => $status !== null ? "GRAPH_HTTP_{$status}" : 'GRAPH_ERROR', 'message' => $bulkOperationService->sanitizeFailureReason($message), ]; } } if (! $dryRun) { $operationRun->update([ 'context' => array_merge($operationRun->context ?? [], [ 'backup_schedule_id' => (int) $scheduleRun->backup_schedule_id, 'backup_schedule_run_id' => (int) $scheduleRun->getKey(), ]), ]); $operationRunService->updateRun( $operationRun, status: 'completed', outcome: $outcome, summaryCounts: $summaryCounts, failures: $failures, ); $operationRun->forceFill([ 'started_at' => $scheduleRun->started_at ?? $operationRun->started_at, 'completed_at' => $scheduleRun->finished_at ?? $operationRun->completed_at, ])->save(); } $reconciled++; } $this->info(sprintf( 'Reconciled %d run(s), skipped %d, failed %d.', $reconciled, $skipped, $failed, )); if ($dryRun) { $this->comment('Dry-run: no changes written.'); } return self::SUCCESS; } /** * @param array $tenantIdentifiers * @return array */ private function resolveTenantIds(array $tenantIdentifiers): array { $tenantIds = []; foreach ($tenantIdentifiers as $identifier) { $tenant = Tenant::query() ->forTenant($identifier) ->first(); if ($tenant) { $tenantIds[] = (int) $tenant->getKey(); } } return array_values(array_unique($tenantIds)); } }