TenantAtlas/app/Console/Commands/TenantpilotReconcileBackupScheduleOperationRuns.php

<?php

namespace App\Console\Commands;

use App\Models\BackupScheduleRun;
use App\Models\OperationRun;
use App\Models\Tenant;
use App\Services\OperationRunService;
use App\Support\OpsUx\RunFailureSanitizer;
use Illuminate\Console\Command;

class TenantpilotReconcileBackupScheduleOperationRuns extends Command
{
    protected $signature = 'tenantpilot:operation-runs:reconcile-backup-schedules
        {--tenant=* : Limit to tenant_id/external_id}
        {--older-than=5 : Only reconcile runs older than N minutes}
        {--dry-run : Do not write changes}';

    protected $description = 'Reconcile stuck backup schedule OperationRuns against BackupScheduleRun status.';

    public function handle(OperationRunService $operationRunService): int
    {
        $tenantIdentifiers = array_values(array_filter((array) $this->option('tenant')));
        $olderThanMinutes = max(0, (int) $this->option('older-than'));
        $dryRun = (bool) $this->option('dry-run');

        $query = OperationRun::query()
            ->whereIn('type', ['backup_schedule.run_now', 'backup_schedule.retry'])
            ->whereIn('status', ['queued', 'running']);

        if ($olderThanMinutes > 0) {
            $query->where('created_at', '<', now()->subMinutes($olderThanMinutes));
        }

        if ($tenantIdentifiers !== []) {
            $tenantIds = $this->resolveTenantIds($tenantIdentifiers);

            if ($tenantIds === []) {
                $this->info('No tenants matched the provided identifiers.');

                return self::SUCCESS;
            }

            $query->whereIn('tenant_id', $tenantIds);
        }

        $reconciled = 0;
        $skipped = 0;
        $failed = 0;

        foreach ($query->cursor() as $operationRun) {
            $backupScheduleRunId = data_get($operationRun->context, 'backup_schedule_run_id');

            if (! is_numeric($backupScheduleRunId)) {
                $skipped++;

                continue;
            }

            $scheduleRun = BackupScheduleRun::query()
                ->whereKey((int) $backupScheduleRunId)
                ->where('tenant_id', $operationRun->tenant_id)
                ->first();

            if (! $scheduleRun) {
                if (! $dryRun) {
                    $operationRunService->updateRun(
                        $operationRun,
                        status: 'completed',
                        outcome: 'failed',
                        failures: [
                            [
                                'code' => 'backup_schedule_run.not_found',
                                'message' => RunFailureSanitizer::sanitizeMessage('Backup schedule run not found.'),
                            ],
                        ],
                    );
                }

                $failed++;

                continue;
            }

            if ($scheduleRun->status === BackupScheduleRun::STATUS_RUNNING) {
                if (! $dryRun) {
                    $operationRunService->updateRun($operationRun, 'running', 'pending');

                    if ($scheduleRun->started_at) {
                        $operationRun->forceFill(['started_at' => $scheduleRun->started_at])->save();
                    }
                }

                $reconciled++;

                continue;
            }

            $outcome = match ($scheduleRun->status) {
                BackupScheduleRun::STATUS_SUCCESS => 'succeeded',
                BackupScheduleRun::STATUS_PARTIAL => 'partially_succeeded',
                BackupScheduleRun::STATUS_SKIPPED => 'succeeded',
                BackupScheduleRun::STATUS_CANCELED => 'failed',
                default => 'failed',
            };

            $summary = is_array($scheduleRun->summary) ? $scheduleRun->summary : [];
            $syncFailures = $summary['sync_failures'] ?? [];

            $policiesTotal = (int) ($summary['policies_total'] ?? 0);
            $policiesBackedUp = (int) ($summary['policies_backed_up'] ?? 0);
            $syncFailuresCount = is_array($syncFailures) ? count($syncFailures) : 0;

            $processed = $policiesBackedUp + $syncFailuresCount;
            if ($policiesTotal > 0) {
                $processed = min($policiesTotal, $processed);
            }

            $summaryCounts = array_filter([
                'total' => $policiesTotal,
                'processed' => $processed,
                'succeeded' => $policiesBackedUp,
                'failed' => $syncFailuresCount,
                'skipped' => $scheduleRun->status === BackupScheduleRun::STATUS_SKIPPED ? 1 : 0,
                'items' => $policiesTotal,
            ], fn (mixed $value): bool => is_int($value) && $value !== 0);

            $failures = [];

            if ($scheduleRun->status === BackupScheduleRun::STATUS_CANCELED) {
                $failures[] = [
                    'code' => 'backup_schedule_run.cancelled',
                    'message' => 'Backup schedule run was cancelled.',
                ];
            }

            if (filled($scheduleRun->error_message) || filled($scheduleRun->error_code)) {
                $failures[] = [
                    'code' => (string) ($scheduleRun->error_code ?: 'backup_schedule_run.error'),
                    'message' => RunFailureSanitizer::sanitizeMessage((string) ($scheduleRun->error_message ?: 'Backup schedule run failed.')),
                ];
            }

            if (is_array($syncFailures)) {
                foreach ($syncFailures as $failure) {
                    if (! is_array($failure)) {
                        continue;
                    }

                    $policyType = (string) ($failure['policy_type'] ?? 'unknown');
                    $status = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null;
                    $errors = $failure['errors'] ?? null;

                    $firstErrorMessage = null;
                    if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) {
                        $firstErrorMessage = $errors[0]['message'] ?? null;
                    }

                    $message = $status !== null
                        ? "{$policyType}: Graph returned {$status}"
                        : "{$policyType}: Graph request failed";

                    if (is_string($firstErrorMessage) && $firstErrorMessage !== '') {
                        $message .= ' - '.trim($firstErrorMessage);
                    }

                    $failures[] = [
                        'code' => $status !== null ? "graph.http_{$status}" : 'graph.error',
                        'message' => RunFailureSanitizer::sanitizeMessage($message),
                    ];
                }
            }

            if (! $dryRun) {
                $operationRun->update([
                    'context' => array_merge($operationRun->context ?? [], [
                        'backup_schedule_id' => (int) $scheduleRun->backup_schedule_id,
                        'backup_schedule_run_id' => (int) $scheduleRun->getKey(),
                    ]),
                ]);

                $operationRunService->updateRun(
                    $operationRun,
                    status: 'completed',
                    outcome: $outcome,
                    summaryCounts: $summaryCounts,
                    failures: $failures,
                );

                $operationRun->forceFill([
                    'started_at' => $scheduleRun->started_at ?? $operationRun->started_at,
                    'completed_at' => $scheduleRun->finished_at ?? $operationRun->completed_at,
                ])->save();
            }

            $reconciled++;
        }

        $this->info(sprintf(
            'Reconciled %d run(s), skipped %d, failed %d.',
            $reconciled,
            $skipped,
            $failed,
        ));

        if ($dryRun) {
            $this->comment('Dry-run: no changes written.');
        }

        return self::SUCCESS;
    }

    /**
     * @param  array<int, string>  $tenantIdentifiers
     * @return array<int>
     */
    private function resolveTenantIds(array $tenantIdentifiers): array
    {
        $tenantIds = [];

        foreach ($tenantIdentifiers as $identifier) {
            $tenant = Tenant::query()
                ->forTenant($identifier)
                ->first();

            if ($tenant) {
                $tenantIds[] = (int) $tenant->getKey();
            }
        }

        return array_values(array_unique($tenantIds));
    }
}