TenantAtlas/app/Jobs/AddPoliciesToBackupSetJob.php

<?php

namespace App\Jobs;

use App\Filament\Resources\BulkOperationRunResource;
use App\Jobs\Middleware\TrackOperationRun;
use App\Models\BackupItem;
use App\Models\BackupSet;
use App\Models\BulkOperationRun;
use App\Models\OperationRun;
use App\Models\Policy;
use App\Models\Tenant;
use App\Services\BulkOperationService;
use App\Services\Intune\FoundationSnapshotService;
use App\Services\Intune\PolicyCaptureOrchestrator;
use App\Services\Intune\SnapshotValidator;
use App\Services\OperationRunService;
use App\Support\OperationRunLinks;
use Filament\Notifications\Notification;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Database\QueryException;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
use Illuminate\Support\Str;
use Throwable;

class AddPoliciesToBackupSetJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    public ?OperationRun $operationRun = null;

    public function __construct(
        public int $bulkRunId,
        public int $backupSetId,
        public bool $includeAssignments,
        public bool $includeScopeTags,
        public bool $includeFoundations,
        ?OperationRun $operationRun = null
    ) {
        $this->operationRun = $operationRun;
    }

    public function middleware(): array
    {
        return [new TrackOperationRun];
    }

    public function handle(
        BulkOperationService $bulkOperationService,
        PolicyCaptureOrchestrator $captureOrchestrator,
        FoundationSnapshotService $foundationSnapshots,
        SnapshotValidator $snapshotValidator,
    ): void {
        $run = BulkOperationRun::with(['tenant', 'user'])->find($this->bulkRunId);

        if (! $run) {
            return;
        }

        $started = BulkOperationRun::query()
            ->whereKey($run->getKey())
            ->where('status', 'pending')
            ->update(['status' => 'running']);

        if ($started === 0) {
            return;
        }

        $run->refresh();

        $tenant = $run->tenant ?? Tenant::query()->find($run->tenant_id);

        try {
            if (! $tenant instanceof Tenant) {
                $this->markRunFailed(
                    bulkOperationService: $bulkOperationService,
                    run: $run,
                    tenant: null,
                    itemId: (string) $this->backupSetId,
                    reasonCode: 'unknown',
                    reason: 'Tenant not found for run.',
                );

                return;
            }

            $backupSet = BackupSet::withTrashed()
                ->where('tenant_id', $tenant->getKey())
                ->whereKey($this->backupSetId)
                ->first();

            if (! $backupSet) {
                $this->markRunFailed(
                    bulkOperationService: $bulkOperationService,
                    run: $run,
                    tenant: $tenant,
                    itemId: (string) $this->backupSetId,
                    reasonCode: 'backup_set_not_found',
                    reason: 'Backup set not found.',
                );

                return;
            }

            if ($backupSet->trashed()) {
                $this->markRunFailed(
                    bulkOperationService: $bulkOperationService,
                    run: $run,
                    tenant: $tenant,
                    itemId: (string) $backupSet->getKey(),
                    reasonCode: 'backup_set_archived',
                    reason: 'Backup set is archived.',
                );

                return;
            }

            $policyIds = $this->extractPolicyIds($run);

            if ($policyIds === []) {
                $bulkOperationService->complete($run);

                if ($this->operationRun) {
                    /** @var OperationRunService $opService */
                    $opService = app(OperationRunService::class);
                    $opService->updateRun($this->operationRun, 'completed', 'succeeded', ['policy_ids' => 0]);
                }

                return;
            }

            if ((int) $run->total_items !== count($policyIds)) {
                $run->update(['total_items' => count($policyIds)]);
            }

            $existingBackupFailures = (array) Arr::get($backupSet->metadata ?? [], 'failures', []);
            $newBackupFailures = [];

            $didMutateBackupSet = false;
            $backupSetItemMutations = 0;
            $foundationMutations = 0;
            $foundationFailures = 0;

            /** @var array<int, int> $activePolicyIds */
            $activePolicyIds = BackupItem::query()
                ->where('backup_set_id', $backupSet->getKey())
                ->whereIn('policy_id', $policyIds)
                ->pluck('policy_id')
                ->filter()
                ->map(fn (mixed $value): int => (int) $value)
                ->values()
                ->all();

            $activePolicyIdSet = array_fill_keys($activePolicyIds, true);

            /** @var EloquentCollection<int, BackupItem> $trashedItems */
            $trashedItems = BackupItem::onlyTrashed()
                ->where('backup_set_id', $backupSet->getKey())
                ->whereIn('policy_id', $policyIds)
                ->get()
                ->keyBy('policy_id');

            /** @var EloquentCollection<int, Policy> $policies */
            $policies = Policy::query()
                ->where('tenant_id', $tenant->getKey())
                ->whereIn('id', $policyIds)
                ->get()
                ->keyBy('id');

            foreach ($policyIds as $policyId) {
                if (isset($activePolicyIdSet[$policyId])) {
                    $bulkOperationService->recordSkippedWithReason(
                        run: $run,
                        itemId: (string) $policyId,
                        reason: 'Already in backup set',
                        reasonCode: 'already_in_backup_set',
                    );

                    continue;
                }

                $trashed = $trashedItems->get($policyId);

                if ($trashed instanceof BackupItem) {
                    $trashed->restore();

                    $activePolicyIdSet[$policyId] = true;
                    $didMutateBackupSet = true;
                    $backupSetItemMutations++;

                    $bulkOperationService->recordSuccess($run);

                    continue;
                }

                $policy = $policies->get($policyId);

                if (! $policy instanceof Policy) {
                    $newBackupFailures[] = [
                        'policy_id' => $policyId,
                        'reason' => $bulkOperationService->sanitizeFailureReason('Policy not found.'),
                        'status' => null,
                        'reason_code' => 'policy_not_found',
                    ];
                    $didMutateBackupSet = true;

                    $bulkOperationService->recordFailure(
                        run: $run,
                        itemId: (string) $policyId,
                        reason: 'Policy not found.',
                        reasonCode: 'policy_not_found',
                    );

                    continue;
                }

                if ($policy->ignored_at) {
                    $bulkOperationService->recordSkippedWithReason(
                        run: $run,
                        itemId: (string) $policyId,
                        reason: 'Policy is ignored locally',
                        reasonCode: 'policy_ignored',
                    );

                    continue;
                }

                try {
                    $captureResult = $captureOrchestrator->capture(
                        policy: $policy,
                        tenant: $tenant,
                        includeAssignments: $this->includeAssignments,
                        includeScopeTags: $this->includeScopeTags,
                        createdBy: $run->user?->email ? Str::limit($run->user->email, 255, '') : null,
                        metadata: [
                            'source' => 'backup',
                            'backup_set_id' => $backupSet->getKey(),
                        ],
                    );
                } catch (Throwable $throwable) {
                    $reason = $bulkOperationService->sanitizeFailureReason($throwable->getMessage());

                    $newBackupFailures[] = [
                        'policy_id' => $policyId,
                        'reason' => $reason,
                        'status' => null,
                        'reason_code' => 'unknown',
                    ];
                    $didMutateBackupSet = true;

                    $bulkOperationService->recordFailure(
                        run: $run,
                        itemId: (string) $policyId,
                        reason: $reason,
                        reasonCode: 'unknown',
                    );

                    continue;
                }

                if (isset($captureResult['failure']) && is_array($captureResult['failure'])) {
                    $failure = $captureResult['failure'];
                    $status = isset($failure['status']) && is_numeric($failure['status']) ? (int) $failure['status'] : null;
                    $reasonCode = $this->mapGraphFailureReasonCode($status);
                    $reason = $bulkOperationService->sanitizeFailureReason((string) ($failure['reason'] ?? 'Graph capture failed.'));

                    $newBackupFailures[] = [
                        'policy_id' => $policyId,
                        'reason' => $reason,
                        'status' => $status,
                        'reason_code' => $reasonCode,
                    ];
                    $didMutateBackupSet = true;

                    $bulkOperationService->recordFailure(
                        run: $run,
                        itemId: (string) $policyId,
                        reason: $reason,
                        reasonCode: $reasonCode,
                    );

                    continue;
                }

                $version = $captureResult['version'] ?? null;
                $captured = $captureResult['captured'] ?? null;

                if (! $version || ! is_array($captured)) {
                    $newBackupFailures[] = [
                        'policy_id' => $policyId,
                        'reason' => $bulkOperationService->sanitizeFailureReason('Capture result missing version payload.'),
                        'status' => null,
                        'reason_code' => 'unknown',
                    ];
                    $didMutateBackupSet = true;

                    $bulkOperationService->recordFailure(
                        run: $run,
                        itemId: (string) $policyId,
                        reason: 'Capture result missing version payload.',
                        reasonCode: 'unknown',
                    );

                    continue;
                }

                $payload = $captured['payload'] ?? [];
                $metadata = is_array($captured['metadata'] ?? null) ? $captured['metadata'] : [];
                $assignments = is_array($captured['assignments'] ?? null) ? $captured['assignments'] : null;
                $scopeTags = is_array($captured['scope_tags'] ?? null) ? $captured['scope_tags'] : null;

                if (! is_array($payload)) {
                    $payload = [];
                }

                $validation = $snapshotValidator->validate($payload);
                $warnings = $validation['warnings'] ?? [];

                $odataWarning = BackupItem::odataTypeWarning($payload, $policy->policy_type, $policy->platform);

                if ($odataWarning) {
                    $warnings[] = $odataWarning;
                }

                if (! empty($warnings)) {
                    $existingWarnings = is_array($metadata['warnings'] ?? null) ? $metadata['warnings'] : [];
                    $metadata['warnings'] = array_values(array_unique(array_merge($existingWarnings, $warnings)));
                }

                if (is_array($scopeTags)) {
                    $metadata['scope_tag_ids'] = $scopeTags['ids'] ?? null;
                    $metadata['scope_tag_names'] = $scopeTags['names'] ?? null;
                }

                try {
                    BackupItem::create([
                        'tenant_id' => $tenant->getKey(),
                        'backup_set_id' => $backupSet->getKey(),
                        'policy_id' => $policy->getKey(),
                        'policy_version_id' => $version->getKey(),
                        'policy_identifier' => $policy->external_id,
                        'policy_type' => $policy->policy_type,
                        'platform' => $policy->platform,
                        'payload' => $payload,
                        'metadata' => $metadata,
                        'assignments' => $assignments,
                    ]);
                } catch (QueryException $exception) {
                    if ((string) $exception->getCode() === '23505') {
                        $bulkOperationService->recordSkippedWithReason(
                            run: $run,
                            itemId: (string) $policyId,
                            reason: 'Already in backup set',
                            reasonCode: 'already_in_backup_set',
                        );

                        continue;
                    }

                    throw $exception;
                }

                $activePolicyIdSet[$policyId] = true;
                $didMutateBackupSet = true;
                $backupSetItemMutations++;

                $bulkOperationService->recordSuccess($run);
            }

            if ($this->includeFoundations) {
                [$foundationOutcome, $foundationFailureEntries] = $this->captureFoundations(
                    bulkOperationService: $bulkOperationService,
                    foundationSnapshots: $foundationSnapshots,
                    tenant: $tenant,
                    backupSet: $backupSet,
                );

                if (($foundationOutcome['created'] ?? 0) > 0 || ($foundationOutcome['restored'] ?? 0) > 0) {
                    $didMutateBackupSet = true;
                    $foundationMutations = (int) $foundationOutcome['created'] + (int) $foundationOutcome['restored'];
                }

                if ($foundationFailureEntries !== []) {
                    $didMutateBackupSet = true;
                    $foundationFailures = count($foundationFailureEntries);
                    $newBackupFailures = array_merge($newBackupFailures, $foundationFailureEntries);

                    foreach ($foundationFailureEntries as $foundationFailure) {
                        $this->appendRunFailure($run, [
                            'type' => 'foundation',
                            'item_id' => (string) ($foundationFailure['foundation_type'] ?? 'foundation'),
                            'reason_code' => (string) ($foundationFailure['reason_code'] ?? 'unknown'),
                            'reason' => (string) ($foundationFailure['reason'] ?? 'Foundation capture failed.'),
                            'status' => $foundationFailure['status'] ?? null,
                        ]);
                    }
                }
            }

            if ($didMutateBackupSet) {
                $allFailures = array_merge($existingBackupFailures, $newBackupFailures);
                $mutations = $backupSetItemMutations + $foundationMutations;

                $backupSetStatus = match (true) {
                    $mutations === 0 && count($allFailures) > 0 => 'failed',
                    count($allFailures) > 0 => 'partial',
                    default => 'completed',
                };

                $backupSet->update([
                    'status' => $backupSetStatus,
                    'item_count' => $backupSet->items()->count(),
                    'completed_at' => now(),
                    'metadata' => ['failures' => $allFailures],
                ]);
            }

            $bulkOperationService->complete($run);

            if ($this->operationRun) {
                /** @var OperationRunService $opService */
                $opService = app(OperationRunService::class);

                $opOutcome = match (true) {
                    $run->status === 'completed' => 'succeeded',
                    $run->status === 'completed_with_errors' => 'partially_succeeded',
                    $run->status === 'failed' => 'failed',
                    default => 'failed'
                };

                $opService->updateRun(
                    $this->operationRun,
                    'completed',
                    $opOutcome,
                    [
                        'policies_added' => $backupSetItemMutations,
                        'foundations_added' => $foundationMutations,
                        'failures' => count($newBackupFailures),
                    ],
                    $newBackupFailures
                );
            }

            if (! $run->user) {
                return;
            }

            $message = "Added {$run->succeeded} policies";
            if ($run->skipped > 0) {
                $message .= " ({$run->skipped} skipped)";
            }
            if ($run->failed > 0) {
                $message .= " ({$run->failed} failed)";
            }

            if ($this->includeFoundations) {
                $message .= ". Foundations: {$foundationMutations} items";

                if ($foundationFailures > 0) {
                    $message .= " ({$foundationFailures} failed)";
                }
            }

            $message .= '.';

            $partial = $run->status === 'completed_with_errors' || $foundationFailures > 0;

            $notification = Notification::make()
                ->title($partial ? 'Add Policies Completed (partial)' : 'Add Policies Completed')
                ->body($message)
                ->actions([
                    \Filament\Actions\Action::make('view_run')
                        ->label('View run')
                        ->url($this->operationRun ? OperationRunLinks::view($this->operationRun, $tenant) : BulkOperationRunResource::getUrl('view', ['record' => $run], tenant: $tenant)),
                ]);

            if ($partial) {
                $notification->warning();
            } else {
                $notification->success();
            }

            $notification
                ->sendToDatabase($run->user)
                ->send();
        } catch (Throwable $throwable) {
            $run->refresh();

            if (in_array($run->status, ['completed', 'completed_with_errors'], true)) {
                throw $throwable;
            }

            $this->markRunFailed(
                bulkOperationService: $bulkOperationService,
                run: $run,
                tenant: $tenant instanceof Tenant ? $tenant : null,
                itemId: (string) $this->backupSetId,
                reasonCode: 'unknown',
                reason: $throwable->getMessage(),
            );

            // TrackOperationRun will catch this throw
            throw $throwable;
        }
    }

    /**
     * @return array<int>
     */
    private function extractPolicyIds(BulkOperationRun $run): array
    {
        $itemIds = $run->item_ids ?? [];

        $policyIds = [];

        if (is_array($itemIds) && array_key_exists('policy_ids', $itemIds) && is_array($itemIds['policy_ids'])) {
            $policyIds = $itemIds['policy_ids'];
        } elseif (is_array($itemIds)) {
            $policyIds = $itemIds;
        }

        $policyIds = array_values(array_unique(array_map('intval', $policyIds)));
        $policyIds = array_values(array_filter($policyIds, fn (int $value): bool => $value > 0));
        sort($policyIds);

        return $policyIds;
    }

    /**
     * @param  array<string, mixed>  $entry
     */
    private function appendRunFailure(BulkOperationRun $run, array $entry): void
    {
        $failures = $run->failures ?? [];

        $failures[] = array_merge([
            'timestamp' => now()->toIso8601String(),
        ], $entry);

        $run->update(['failures' => $failures]);
    }

    private function markRunFailed(
        BulkOperationService $bulkOperationService,
        BulkOperationRun $run,
        ?Tenant $tenant,
        string $itemId,
        string $reasonCode,
        string $reason,
    ): void {
        $reason = $bulkOperationService->sanitizeFailureReason($reason);

        $this->appendRunFailure($run, [
            'type' => 'run',
            'item_id' => $itemId,
            'reason_code' => $reasonCode,
            'reason' => $reason,
        ]);

        try {
            $bulkOperationService->fail($run, $reason);
        } catch (Throwable) {
            $run->update(['status' => 'failed']);
        }

        if ($this->operationRun) {
            /** @var OperationRunService $opService */
            $opService = app(OperationRunService::class);
            $opService->updateRun(
                $this->operationRun,
                'completed',
                'failed',
                ['failure_reason' => $reason],
                [['code' => $reasonCode, 'message' => $reason]]
            );
        }

        $this->notifyRunFailed($run, $tenant, $reason);
    }

    private function notifyRunFailed(BulkOperationRun $run, ?Tenant $tenant, string $reason): void
    {
        if (! $run->user) {
            return;
        }

        $notification = Notification::make()
            ->title('Add Policies Failed')
            ->body($reason);

        if ($tenant instanceof Tenant) {
            $notification->actions([
                \Filament\Actions\Action::make('view_run')
                    ->label('View run')
                    ->url($this->operationRun ? OperationRunLinks::view($this->operationRun, $tenant) : BulkOperationRunResource::getUrl('view', ['record' => $run], tenant: $tenant)),
            ]);
        }

        $notification
            ->danger()
            ->sendToDatabase($run->user)
            ->send();
    }

    private function mapGraphFailureReasonCode(?int $status): string
    {
        return match (true) {
            $status === 403 => 'graph_forbidden',
            in_array($status, [429, 503], true) => 'graph_throttled',
            in_array($status, [408, 500, 502, 504], true) => 'graph_transient',
            default => 'unknown',
        };
    }

    /**
     * @return array{0:array{created:int,restored:int,failures:array<int,array{foundation_type:string,reason:string,status:int|string|null,reason_code:string}>},1:array<int,array{foundation_type:string,reason:string,status:int|string|null,reason_code:string}>}
     */
    private function captureFoundations(
        BulkOperationService $bulkOperationService,
        FoundationSnapshotService $foundationSnapshots,
        Tenant $tenant,
        BackupSet $backupSet,
    ): array {
        $types = config('tenantpilot.foundation_types', []);
        $created = 0;
        $restored = 0;
        $failures = [];

        foreach ($types as $typeConfig) {
            $foundationType = $typeConfig['type'] ?? null;

            if (! is_string($foundationType) || $foundationType === '') {
                continue;
            }

            $result = $foundationSnapshots->fetchAll($tenant, $foundationType);

            foreach (($result['failures'] ?? []) as $failure) {
                if (! is_array($failure)) {
                    continue;
                }

                $status = isset($failure['status']) && is_numeric($failure['status']) ? (int) $failure['status'] : null;
                $reasonCode = $this->mapGraphFailureReasonCode($status);
                $reason = $bulkOperationService->sanitizeFailureReason((string) ($failure['reason'] ?? 'Foundation capture failed.'));

                $failures[] = [
                    'foundation_type' => $foundationType,
                    'reason' => $reason,
                    'status' => $status,
                    'reason_code' => $reasonCode,
                ];
            }

            foreach (($result['items'] ?? []) as $snapshot) {
                if (! is_array($snapshot)) {
                    continue;
                }

                $sourceId = $snapshot['source_id'] ?? null;

                if (! is_string($sourceId) || $sourceId === '') {
                    continue;
                }

                $existing = BackupItem::withTrashed()
                    ->where('backup_set_id', $backupSet->getKey())
                    ->where('policy_type', $foundationType)
                    ->where('policy_identifier', $sourceId)
                    ->first();

                if ($existing) {
                    if ($existing->trashed()) {
                        $existing->restore();
                        $restored++;
                    }

                    continue;
                }

                BackupItem::create([
                    'tenant_id' => $tenant->getKey(),
                    'backup_set_id' => $backupSet->getKey(),
                    'policy_id' => null,
                    'policy_identifier' => $sourceId,
                    'policy_type' => $foundationType,
                    'platform' => $typeConfig['platform'] ?? null,
                    'payload' => $snapshot['payload'] ?? [],
                    'metadata' => $snapshot['metadata'] ?? [],
                ]);

                $created++;
            }
        }

        return [
            [
                'created' => $created,
                'restored' => $restored,
                'failures' => $failures,
            ],
            $failures,
        ];
    }
}