TenantAtlas/app/Jobs/RunBackupScheduleJob.php

<?php

namespace App\Jobs;

use App\Jobs\Middleware\TrackOperationRun;
use App\Models\BackupSchedule;
use App\Models\OperationRun;
use App\Models\Tenant;
use App\Models\User;
use App\Services\BackupScheduling\PolicyTypeResolver;
use App\Services\BackupScheduling\RunErrorMapper;
use App\Services\BackupScheduling\ScheduleTimeService;
use App\Services\Intune\AuditLogger;
use App\Services\Intune\BackupService;
use App\Services\Intune\PolicySyncService;
use App\Services\OperationRunService;
use App\Support\OperationRunLinks;
use App\Support\OperationRunOutcome;
use Carbon\CarbonImmutable;
use Filament\Actions\Action;
use Filament\Notifications\Notification;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Cache;

class RunBackupScheduleJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    private const string STATUS_RUNNING = 'running';

    private const string STATUS_SUCCESS = 'success';

    private const string STATUS_PARTIAL = 'partial';

    private const string STATUS_FAILED = 'failed';

    private const string STATUS_SKIPPED = 'skipped';

    public int $tries = 3;

    public ?OperationRun $operationRun = null;

    public function __construct(
        public int $backupScheduleRunId,
        ?OperationRun $operationRun = null,
        public ?int $backupScheduleId = null,
    ) {
        $this->operationRun = $operationRun;
    }

    public function middleware(): array
    {
        return [new TrackOperationRun];
    }

    public function handle(
        PolicySyncService $policySyncService,
        BackupService $backupService,
        PolicyTypeResolver $policyTypeResolver,
        ScheduleTimeService $scheduleTimeService,
        AuditLogger $auditLogger,
        RunErrorMapper $errorMapper,
    ): void {
        if (! $this->operationRun) {
            $this->fail(new \RuntimeException('OperationRun context is required for RunBackupScheduleJob.'));

            return;
        }

        $backupScheduleId = $this->resolveBackupScheduleId();

        if ($backupScheduleId <= 0) {
            $this->markOperationRunFailed(
                run: $this->operationRun,
                summaryCounts: [],
                reasonCode: 'schedule_not_provided',
                reason: 'No backup schedule was provided for this run.',
            );

            return;
        }

        $schedule = BackupSchedule::query()
            ->with('tenant')
            ->find($backupScheduleId);

        if (! $schedule instanceof BackupSchedule) {
            $this->markOperationRunFailed(
                run: $this->operationRun,
                summaryCounts: [],
                reasonCode: 'schedule_not_found',
                reason: 'Schedule not found.',
            );

            return;
        }

        $tenant = $schedule->tenant;

        if (! $tenant instanceof Tenant) {
            $this->markOperationRunFailed(
                run: $this->operationRun,
                summaryCounts: [],
                reasonCode: 'tenant_not_found',
                reason: 'Tenant not found.',
            );

            return;
        }

        /** @var OperationRunService $operationRunService */
        $operationRunService = app(OperationRunService::class);

        $this->operationRun->update([
            'context' => array_merge($this->operationRun->context ?? [], [
                'backup_schedule_id' => (int) $schedule->getKey(),
            ]),
        ]);

        if ($this->operationRun->status === 'queued') {
            $operationRunService->updateRun($this->operationRun, 'running');
        }

        $lock = Cache::lock("backup_schedule:{$schedule->id}", 900);

        if (! $lock->get()) {
            $nowUtc = CarbonImmutable::now('UTC');

            $this->finishSchedule(
                schedule: $schedule,
                status: self::STATUS_SKIPPED,
                scheduleTimeService: $scheduleTimeService,
                nowUtc: $nowUtc,
            );

            $operationRunService->updateRun(
                $this->operationRun,
                status: 'completed',
                outcome: OperationRunOutcome::Blocked->value,
                summaryCounts: [
                    'total' => 0,
                    'processed' => 0,
                    'failed' => 0,
                    'skipped' => 1,
                ],
                failures: [
                    [
                        'code' => 'concurrent_run',
                        'message' => 'Another run is already in progress for this schedule.',
                    ],
                ],
            );

            $this->notifyScheduleRunFinished(
                tenant: $tenant,
                schedule: $schedule,
                status: self::STATUS_SKIPPED,
                errorMessage: 'Another run is already in progress for this schedule.',
            );

            $auditLogger->log(
                tenant: $tenant,
                action: 'backup_schedule.run_skipped',
                context: [
                    'metadata' => [
                        'backup_schedule_id' => $schedule->id,
                        'operation_run_id' => $this->operationRun->getKey(),
                        'reason' => 'concurrent_run',
                    ],
                ],
                resourceType: 'operation_run',
                resourceId: (string) $this->operationRun->getKey(),
                status: 'partial'
            );

            return;
        }

        try {
            $nowUtc = CarbonImmutable::now('UTC');

            $this->notifyScheduleRunStarted(tenant: $tenant, schedule: $schedule);

            $auditLogger->log(
                tenant: $tenant,
                action: 'backup_schedule.run_started',
                context: [
                    'metadata' => [
                        'backup_schedule_id' => $schedule->id,
                        'operation_run_id' => $this->operationRun->getKey(),
                    ],
                ],
                resourceType: 'operation_run',
                resourceId: (string) $this->operationRun->getKey(),
                status: 'success'
            );

            $runtime = $policyTypeResolver->resolveRuntime((array) ($schedule->policy_types ?? []));
            $validTypes = $runtime['valid'];
            $unknownTypes = $runtime['unknown'];

            if (empty($validTypes)) {
                $this->finishSchedule(
                    schedule: $schedule,
                    status: self::STATUS_SKIPPED,
                    scheduleTimeService: $scheduleTimeService,
                    nowUtc: $nowUtc,
                );

                $operationRunService->updateRun(
                    $this->operationRun,
                    status: 'completed',
                    outcome: OperationRunOutcome::Blocked->value,
                    summaryCounts: [
                        'total' => 0,
                        'processed' => 0,
                        'failed' => 0,
                        'skipped' => 1,
                    ],
                    failures: [
                        [
                            'code' => 'unknown_policy_type',
                            'message' => 'All configured policy types are unknown.',
                        ],
                    ],
                );

                $this->notifyScheduleRunFinished(
                    tenant: $tenant,
                    schedule: $schedule,
                    status: self::STATUS_SKIPPED,
                    errorMessage: 'All configured policy types are unknown.',
                );

                return;
            }

            $supported = array_values(array_filter(
                config('tenantpilot.supported_policy_types', []),
                fn (array $typeConfig): bool => in_array($typeConfig['type'] ?? null, $validTypes, true),
            ));

            $syncReport = $policySyncService->syncPoliciesWithReport($tenant, $supported);

            $policyIds = $syncReport['synced'] ?? [];
            $syncFailures = $syncReport['failures'] ?? [];

            $backupSet = $backupService->createBackupSet(
                tenant: $tenant,
                policyIds: $policyIds,
                actorEmail: null,
                actorName: null,
                name: 'Scheduled backup: '.$schedule->name,
                includeAssignments: false,
                includeScopeTags: false,
                includeFoundations: (bool) ($schedule->include_foundations ?? false),
            );

            $status = match ($backupSet->status) {
                'completed' => self::STATUS_SUCCESS,
                'partial' => self::STATUS_PARTIAL,
                'failed' => self::STATUS_FAILED,
                default => self::STATUS_SUCCESS,
            };

            $errorCode = null;
            $errorMessage = null;

            if (! empty($unknownTypes)) {
                $status = self::STATUS_PARTIAL;
                $errorCode = 'UNKNOWN_POLICY_TYPE';
                $errorMessage = 'Some configured policy types are unknown and were skipped.';
            }

            $policiesTotal = count($policyIds);
            $policiesBackedUp = (int) ($backupSet->item_count ?? 0);
            $failedCount = max(0, $policiesTotal - $policiesBackedUp);

            $summaryCounts = [
                'total' => $policiesTotal,
                'processed' => $policiesTotal,
                'succeeded' => $policiesBackedUp,
                'failed' => $failedCount,
                'skipped' => 0,
                'created' => 1,
                'updated' => $policiesBackedUp,
                'items' => $policiesTotal,
            ];

            $failures = [];

            if (is_string($errorMessage) && $errorMessage !== '') {
                $failures[] = [
                    'code' => strtolower((string) ($errorCode ?: 'backup_schedule_error')),
                    'message' => $errorMessage,
                ];
            }

            if (is_array($syncFailures)) {
                foreach ($syncFailures as $failure) {
                    if (! is_array($failure)) {
                        continue;
                    }

                    $policyType = (string) ($failure['policy_type'] ?? 'unknown');
                    $httpStatus = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null;
                    $errors = $failure['errors'] ?? null;

                    $firstErrorMessage = null;
                    if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) {
                        $firstErrorMessage = $errors[0]['message'] ?? null;
                    }

                    $message = $httpStatus !== null
                        ? "{$policyType}: Graph returned {$httpStatus}"
                        : "{$policyType}: Graph request failed";

                    if (is_string($firstErrorMessage) && $firstErrorMessage !== '') {
                        $message .= ' - '.trim($firstErrorMessage);
                    }

                    $failures[] = [
                        'code' => $httpStatus !== null ? 'graph_http_'.(string) $httpStatus : 'graph_error',
                        'message' => $message,
                    ];
                }
            }

            $this->operationRun->update([
                'context' => array_merge($this->operationRun->context ?? [], [
                    'backup_schedule_id' => (int) $schedule->getKey(),
                    'backup_set_id' => (int) $backupSet->getKey(),
                ]),
            ]);

            $outcome = match ($status) {
                self::STATUS_SUCCESS => OperationRunOutcome::Succeeded->value,
                self::STATUS_PARTIAL => OperationRunOutcome::PartiallySucceeded->value,
                self::STATUS_SKIPPED => OperationRunOutcome::Blocked->value,
                default => OperationRunOutcome::Failed->value,
            };

            $operationRunService->updateRun(
                $this->operationRun,
                status: 'completed',
                outcome: $outcome,
                summaryCounts: $summaryCounts,
                failures: $failures,
            );

            $this->finishSchedule(
                schedule: $schedule,
                status: $status,
                scheduleTimeService: $scheduleTimeService,
                nowUtc: $nowUtc,
            );

            $this->notifyScheduleRunFinished(
                tenant: $tenant,
                schedule: $schedule,
                status: $status,
                errorMessage: $errorMessage,
            );

            if (in_array($status, [self::STATUS_SUCCESS, self::STATUS_PARTIAL], true)) {
                Bus::dispatch(new ApplyBackupScheduleRetentionJob((int) $schedule->getKey()));
            }

            $auditLogger->log(
                tenant: $tenant,
                action: 'backup_schedule.run_finished',
                context: [
                    'metadata' => [
                        'backup_schedule_id' => $schedule->id,
                        'operation_run_id' => $this->operationRun->getKey(),
                        'status' => $status,
                        'error_code' => $errorCode,
                    ],
                ],
                resourceType: 'operation_run',
                resourceId: (string) $this->operationRun->getKey(),
                status: in_array($status, [self::STATUS_SUCCESS], true) ? 'success' : 'partial'
            );
        } catch (\Throwable $throwable) {
            $attempt = (int) method_exists($this, 'attempts') ? $this->attempts() : 1;
            $mapped = $errorMapper->map($throwable, $attempt, $this->tries);

            if ($mapped['shouldRetry']) {
                $operationRunService->updateRun($this->operationRun, 'running', OperationRunOutcome::Pending->value);

                $this->release($mapped['delay']);

                return;
            }

            $nowUtc = CarbonImmutable::now('UTC');

            $this->finishSchedule(
                schedule: $schedule,
                status: self::STATUS_FAILED,
                scheduleTimeService: $scheduleTimeService,
                nowUtc: $nowUtc,
            );

            $operationRunService->updateRun(
                $this->operationRun,
                status: 'completed',
                outcome: OperationRunOutcome::Failed->value,
                summaryCounts: [
                    'total' => 0,
                    'processed' => 0,
                    'failed' => 1,
                ],
                failures: [
                    [
                        'code' => strtolower((string) $mapped['error_code']),
                        'message' => (string) $mapped['error_message'],
                    ],
                ],
            );

            $this->notifyScheduleRunFinished(
                tenant: $tenant,
                schedule: $schedule,
                status: self::STATUS_FAILED,
                errorMessage: (string) $mapped['error_message'],
            );

            $auditLogger->log(
                tenant: $tenant,
                action: 'backup_schedule.run_failed',
                context: [
                    'metadata' => [
                        'backup_schedule_id' => $schedule->id,
                        'operation_run_id' => $this->operationRun->getKey(),
                        'error_code' => $mapped['error_code'],
                    ],
                ],
                resourceType: 'operation_run',
                resourceId: (string) $this->operationRun->getKey(),
                status: 'failed'
            );
        } finally {
            optional($lock)->release();
        }
    }

    private function resolveBackupScheduleId(): int
    {
        if ($this->backupScheduleId !== null && $this->backupScheduleId > 0) {
            return $this->backupScheduleId;
        }

        $contextScheduleId = data_get($this->operationRun?->context, 'backup_schedule_id');

        return is_numeric($contextScheduleId) ? (int) $contextScheduleId : 0;
    }

    private function notifyScheduleRunStarted(Tenant $tenant, BackupSchedule $schedule): void
    {
        $userId = $this->operationRun?->user_id;

        if (! $userId) {
            return;
        }

        $user = User::query()->find($userId);

        if (! $user instanceof User) {
            return;
        }

        Notification::make()
            ->title('Backup started')
            ->body(sprintf('Schedule "%s" has started.', $schedule->name))
            ->info()
            ->actions([
                Action::make('view_run')
                    ->label('View run')
                    ->url(OperationRunLinks::view($this->operationRun, $tenant)),
            ])
            ->sendToDatabase($user);
    }

    private function notifyScheduleRunFinished(
        Tenant $tenant,
        BackupSchedule $schedule,
        string $status,
        ?string $errorMessage,
    ): void {
        $userId = $this->operationRun?->user_id;

        if (! $userId) {
            return;
        }

        $user = User::query()->find($userId);

        if (! $user instanceof User) {
            return;
        }

        $title = match ($status) {
            self::STATUS_SUCCESS => 'Backup completed',
            self::STATUS_PARTIAL => 'Backup completed (partial)',
            self::STATUS_SKIPPED => 'Backup skipped',
            default => 'Backup failed',
        };

        $notification = Notification::make()
            ->title($title)
            ->body(sprintf('Schedule "%s" finished with status: %s.', $schedule->name, $status));

        if (is_string($errorMessage) && $errorMessage !== '') {
            $notification->body($notification->getBody()."\n".$errorMessage);
        }

        match ($status) {
            self::STATUS_SUCCESS => $notification->success(),
            self::STATUS_PARTIAL, self::STATUS_SKIPPED => $notification->warning(),
            default => $notification->danger(),
        };

        $notification
            ->actions([
                Action::make('view_run')
                    ->label('View run')
                    ->url(OperationRunLinks::view($this->operationRun, $tenant)),
            ])
            ->sendToDatabase($user);
    }

    private function finishSchedule(
        BackupSchedule $schedule,
        string $status,
        ScheduleTimeService $scheduleTimeService,
        CarbonImmutable $nowUtc,
    ): void {
        $schedule->forceFill([
            'last_run_at' => $nowUtc,
            'last_run_status' => $status,
            'next_run_at' => $scheduleTimeService->nextRunFor($schedule, $nowUtc),
        ])->saveQuietly();
    }

    private function markOperationRunFailed(
        OperationRun $run,
        array $summaryCounts,
        string $reasonCode,
        string $reason,
    ): void {
        /** @var OperationRunService $operationRunService */
        $operationRunService = app(OperationRunService::class);

        $operationRunService->updateRun(
            $run,
            status: 'completed',
            outcome: OperationRunOutcome::Failed->value,
            summaryCounts: $summaryCounts,
            failures: [
                [
                    'code' => $reasonCode,
                    'message' => $reason,
                ],
            ],
        );
    }
}