feat(052): async add policies to backup set

app/Jobs/AddPoliciesToBackupSetJob.php

@@ -0,0 +1,573 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Filament\Resources\BulkOperationRunResource;
+use App\Models\BackupItem;
+use App\Models\BackupSet;
+use App\Models\BulkOperationRun;
+use App\Models\Policy;
+use App\Models\Tenant;
+use App\Services\BulkOperationService;
+use App\Services\Intune\FoundationSnapshotService;
+use App\Services\Intune\PolicyCaptureOrchestrator;
+use App\Services\Intune\SnapshotValidator;
+use Filament\Notifications\Notification;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Database\Eloquent\Collection as EloquentCollection;
+use Illuminate\Database\QueryException;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Str;
+use Throwable;
+
+class AddPoliciesToBackupSetJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function __construct(
+        public int $bulkRunId,
+        public int $backupSetId,
+        public bool $includeAssignments,
+        public bool $includeScopeTags,
+        public bool $includeFoundations,
+    ) {}
+
+    public function handle(
+        BulkOperationService $bulkOperationService,
+        PolicyCaptureOrchestrator $captureOrchestrator,
+        FoundationSnapshotService $foundationSnapshots,
+        SnapshotValidator $snapshotValidator,
+    ): void {
+        $run = BulkOperationRun::with(['tenant', 'user'])->find($this->bulkRunId);
+
+        if (! $run || $run->status !== 'pending') {
+            return;
+        }
+
+        $bulkOperationService->start($run);
+
+        $tenant = $run->tenant ?? Tenant::query()->find($run->tenant_id);
+
+        if (! $tenant instanceof Tenant) {
+            $this->appendRunFailure($run, [
+                'type' => 'run',
+                'item_id' => (string) $this->backupSetId,
+                'reason_code' => 'backup_set_not_found',
+                'reason' => $bulkOperationService->sanitizeFailureReason('Tenant not found for run.'),
+            ]);
+
+            $bulkOperationService->fail($run, 'Tenant not found for run.');
+
+            return;
+        }
+
+        $backupSet = BackupSet::withTrashed()
+            ->where('tenant_id', $tenant->getKey())
+            ->whereKey($this->backupSetId)
+            ->first();
+
+        if (! $backupSet) {
+            $this->appendRunFailure($run, [
+                'type' => 'run',
+                'item_id' => (string) $this->backupSetId,
+                'reason_code' => 'backup_set_not_found',
+                'reason' => $bulkOperationService->sanitizeFailureReason('Backup set not found.'),
+            ]);
+
+            $bulkOperationService->fail($run, 'Backup set not found.');
+
+            return;
+        }
+
+        if ($backupSet->trashed()) {
+            $this->appendRunFailure($run, [
+                'type' => 'run',
+                'item_id' => (string) $backupSet->getKey(),
+                'reason_code' => 'backup_set_archived',
+                'reason' => $bulkOperationService->sanitizeFailureReason('Backup set is archived.'),
+            ]);
+
+            $bulkOperationService->fail($run, 'Backup set is archived.');
+
+            return;
+        }
+
+        $policyIds = $this->extractPolicyIds($run);
+
+        if ($policyIds === []) {
+            $bulkOperationService->complete($run);
+
+            return;
+        }
+
+        if ((int) $run->total_items !== count($policyIds)) {
+            $run->update(['total_items' => count($policyIds)]);
+        }
+
+        $existingBackupFailures = (array) Arr::get($backupSet->metadata ?? [], 'failures', []);
+        $newBackupFailures = [];
+
+        $didMutateBackupSet = false;
+        $backupSetItemMutations = 0;
+        $foundationMutations = 0;
+        $foundationFailures = 0;
+
+        /** @var array<int, int> $activePolicyIds */
+        $activePolicyIds = BackupItem::query()
+            ->where('backup_set_id', $backupSet->getKey())
+            ->whereIn('policy_id', $policyIds)
+            ->pluck('policy_id')
+            ->filter()
+            ->map(fn (mixed $value): int => (int) $value)
+            ->values()
+            ->all();
+
+        $activePolicyIdSet = array_fill_keys($activePolicyIds, true);
+
+        /** @var EloquentCollection<int, BackupItem> $trashedItems */
+        $trashedItems = BackupItem::onlyTrashed()
+            ->where('backup_set_id', $backupSet->getKey())
+            ->whereIn('policy_id', $policyIds)
+            ->get()
+            ->keyBy('policy_id');
+
+        /** @var EloquentCollection<int, Policy> $policies */
+        $policies = Policy::query()
+            ->where('tenant_id', $tenant->getKey())
+            ->whereIn('id', $policyIds)
+            ->get()
+            ->keyBy('id');
+
+        foreach ($policyIds as $policyId) {
+            if (isset($activePolicyIdSet[$policyId])) {
+                $bulkOperationService->recordSkippedWithReason(
+                    run: $run,
+                    itemId: (string) $policyId,
+                    reason: 'Already in backup set',
+                    reasonCode: 'already_in_backup_set',
+                );
+
+                continue;
+            }
+
+            $trashed = $trashedItems->get($policyId);
+
+            if ($trashed instanceof BackupItem) {
+                $trashed->restore();
+
+                $activePolicyIdSet[$policyId] = true;
+                $didMutateBackupSet = true;
+                $backupSetItemMutations++;
+
+                $bulkOperationService->recordSuccess($run);
+
+                continue;
+            }
+
+            $policy = $policies->get($policyId);
+
+            if (! $policy instanceof Policy) {
+                $newBackupFailures[] = [
+                    'policy_id' => $policyId,
+                    'reason' => $bulkOperationService->sanitizeFailureReason('Policy not found.'),
+                    'status' => null,
+                    'reason_code' => 'policy_not_found',
+                ];
+                $didMutateBackupSet = true;
+
+                $bulkOperationService->recordFailure(
+                    run: $run,
+                    itemId: (string) $policyId,
+                    reason: 'Policy not found.',
+                    reasonCode: 'policy_not_found',
+                );
+
+                continue;
+            }
+
+            if ($policy->ignored_at) {
+                $bulkOperationService->recordSkippedWithReason(
+                    run: $run,
+                    itemId: (string) $policyId,
+                    reason: 'Policy is ignored locally',
+                    reasonCode: 'policy_ignored',
+                );
+
+                continue;
+            }
+
+            try {
+                $captureResult = $captureOrchestrator->capture(
+                    policy: $policy,
+                    tenant: $tenant,
+                    includeAssignments: $this->includeAssignments,
+                    includeScopeTags: $this->includeScopeTags,
+                    createdBy: $run->user?->email ? Str::limit($run->user->email, 255, '') : null,
+                    metadata: [
+                        'source' => 'backup',
+                        'backup_set_id' => $backupSet->getKey(),
+                    ],
+                );
+            } catch (Throwable $throwable) {
+                $reason = $bulkOperationService->sanitizeFailureReason($throwable->getMessage());
+
+                $newBackupFailures[] = [
+                    'policy_id' => $policyId,
+                    'reason' => $reason,
+                    'status' => null,
+                    'reason_code' => 'unknown',
+                ];
+                $didMutateBackupSet = true;
+
+                $bulkOperationService->recordFailure(
+                    run: $run,
+                    itemId: (string) $policyId,
+                    reason: $reason,
+                    reasonCode: 'unknown',
+                );
+
+                continue;
+            }
+
+            if (isset($captureResult['failure']) && is_array($captureResult['failure'])) {
+                $failure = $captureResult['failure'];
+                $status = isset($failure['status']) && is_numeric($failure['status']) ? (int) $failure['status'] : null;
+                $reasonCode = $this->mapGraphFailureReasonCode($status);
+                $reason = $bulkOperationService->sanitizeFailureReason((string) ($failure['reason'] ?? 'Graph capture failed.'));
+
+                $newBackupFailures[] = [
+                    'policy_id' => $policyId,
+                    'reason' => $reason,
+                    'status' => $status,
+                    'reason_code' => $reasonCode,
+                ];
+                $didMutateBackupSet = true;
+
+                $bulkOperationService->recordFailure(
+                    run: $run,
+                    itemId: (string) $policyId,
+                    reason: $reason,
+                    reasonCode: $reasonCode,
+                );
+
+                continue;
+            }
+
+            $version = $captureResult['version'] ?? null;
+            $captured = $captureResult['captured'] ?? null;
+
+            if (! $version || ! is_array($captured)) {
+                $newBackupFailures[] = [
+                    'policy_id' => $policyId,
+                    'reason' => $bulkOperationService->sanitizeFailureReason('Capture result missing version payload.'),
+                    'status' => null,
+                    'reason_code' => 'unknown',
+                ];
+                $didMutateBackupSet = true;
+
+                $bulkOperationService->recordFailure(
+                    run: $run,
+                    itemId: (string) $policyId,
+                    reason: 'Capture result missing version payload.',
+                    reasonCode: 'unknown',
+                );
+
+                continue;
+            }
+
+            $payload = $captured['payload'] ?? [];
+            $metadata = is_array($captured['metadata'] ?? null) ? $captured['metadata'] : [];
+            $assignments = is_array($captured['assignments'] ?? null) ? $captured['assignments'] : null;
+            $scopeTags = is_array($captured['scope_tags'] ?? null) ? $captured['scope_tags'] : null;
+
+            if (! is_array($payload)) {
+                $payload = [];
+            }
+
+            $validation = $snapshotValidator->validate($payload);
+            $warnings = $validation['warnings'] ?? [];
+
+            $odataWarning = BackupItem::odataTypeWarning($payload, $policy->policy_type, $policy->platform);
+
+            if ($odataWarning) {
+                $warnings[] = $odataWarning;
+            }
+
+            if (! empty($warnings)) {
+                $existingWarnings = is_array($metadata['warnings'] ?? null) ? $metadata['warnings'] : [];
+                $metadata['warnings'] = array_values(array_unique(array_merge($existingWarnings, $warnings)));
+            }
+
+            if (is_array($scopeTags)) {
+                $metadata['scope_tag_ids'] = $scopeTags['ids'] ?? null;
+                $metadata['scope_tag_names'] = $scopeTags['names'] ?? null;
+            }
+
+            try {
+                BackupItem::create([
+                    'tenant_id' => $tenant->getKey(),
+                    'backup_set_id' => $backupSet->getKey(),
+                    'policy_id' => $policy->getKey(),
+                    'policy_version_id' => $version->getKey(),
+                    'policy_identifier' => $policy->external_id,
+                    'policy_type' => $policy->policy_type,
+                    'platform' => $policy->platform,
+                    'payload' => $payload,
+                    'metadata' => $metadata,
+                    'assignments' => $assignments,
+                ]);
+            } catch (QueryException $exception) {
+                if ((string) $exception->getCode() === '23505') {
+                    $bulkOperationService->recordSkippedWithReason(
+                        run: $run,
+                        itemId: (string) $policyId,
+                        reason: 'Already in backup set',
+                        reasonCode: 'already_in_backup_set',
+                    );
+
+                    continue;
+                }
+
+                throw $exception;
+            }
+
+            $activePolicyIdSet[$policyId] = true;
+            $didMutateBackupSet = true;
+            $backupSetItemMutations++;
+
+            $bulkOperationService->recordSuccess($run);
+        }
+
+        if ($this->includeFoundations) {
+            [$foundationOutcome, $foundationFailureEntries] = $this->captureFoundations(
+                bulkOperationService: $bulkOperationService,
+                foundationSnapshots: $foundationSnapshots,
+                tenant: $tenant,
+                backupSet: $backupSet,
+            );
+
+            if (($foundationOutcome['created'] ?? 0) > 0 || ($foundationOutcome['restored'] ?? 0) > 0) {
+                $didMutateBackupSet = true;
+                $foundationMutations = (int) $foundationOutcome['created'] + (int) $foundationOutcome['restored'];
+            }
+
+            if ($foundationFailureEntries !== []) {
+                $didMutateBackupSet = true;
+                $foundationFailures = count($foundationFailureEntries);
+                $newBackupFailures = array_merge($newBackupFailures, $foundationFailureEntries);
+
+                foreach ($foundationFailureEntries as $foundationFailure) {
+                    $this->appendRunFailure($run, [
+                        'type' => 'foundation',
+                        'item_id' => (string) ($foundationFailure['foundation_type'] ?? 'foundation'),
+                        'reason_code' => (string) ($foundationFailure['reason_code'] ?? 'unknown'),
+                        'reason' => (string) ($foundationFailure['reason'] ?? 'Foundation capture failed.'),
+                        'status' => $foundationFailure['status'] ?? null,
+                    ]);
+                }
+            }
+        }
+
+        if ($didMutateBackupSet) {
+            $allFailures = array_merge($existingBackupFailures, $newBackupFailures);
+            $mutations = $backupSetItemMutations + $foundationMutations;
+
+            $backupSetStatus = match (true) {
+                $mutations === 0 && count($allFailures) > 0 => 'failed',
+                count($allFailures) > 0 => 'partial',
+                default => 'completed',
+            };
+
+            $backupSet->update([
+                'status' => $backupSetStatus,
+                'item_count' => $backupSet->items()->count(),
+                'completed_at' => now(),
+                'metadata' => ['failures' => $allFailures],
+            ]);
+        }
+
+        $bulkOperationService->complete($run);
+
+        if (! $run->user) {
+            return;
+        }
+
+        $message = "Added {$run->succeeded} policies";
+        if ($run->skipped > 0) {
+            $message .= " ({$run->skipped} skipped)";
+        }
+        if ($run->failed > 0) {
+            $message .= " ({$run->failed} failed)";
+        }
+
+        if ($this->includeFoundations) {
+            $message .= ". Foundations: {$foundationMutations} items";
+
+            if ($foundationFailures > 0) {
+                $message .= " ({$foundationFailures} failed)";
+            }
+        }
+
+        $message .= '.';
+
+        $notification = Notification::make()
+            ->title($run->failed > 0 ? 'Add Policies Completed (partial)' : 'Add Policies Completed')
+            ->body($message)
+            ->actions([
+                \Filament\Actions\Action::make('view_run')
+                    ->label('View run')
+                    ->url(BulkOperationRunResource::getUrl('view', ['record' => $run], tenant: $tenant)),
+            ]);
+
+        if ($run->failed > 0) {
+            $notification->warning();
+        } else {
+            $notification->success();
+        }
+
+        $notification
+            ->sendToDatabase($run->user)
+            ->send();
+    }
+
+    /**
+     * @return array<int>
+     */
+    private function extractPolicyIds(BulkOperationRun $run): array
+    {
+        $itemIds = $run->item_ids ?? [];
+
+        $policyIds = [];
+
+        if (is_array($itemIds) && array_key_exists('policy_ids', $itemIds) && is_array($itemIds['policy_ids'])) {
+            $policyIds = $itemIds['policy_ids'];
+        } elseif (is_array($itemIds)) {
+            $policyIds = $itemIds;
+        }
+
+        $policyIds = array_values(array_unique(array_map('intval', $policyIds)));
+        $policyIds = array_values(array_filter($policyIds, fn (int $value): bool => $value > 0));
+        sort($policyIds);
+
+        return $policyIds;
+    }
+
+    /**
+     * @param  array<string, mixed>  $entry
+     */
+    private function appendRunFailure(BulkOperationRun $run, array $entry): void
+    {
+        $failures = $run->failures ?? [];
+
+        $failures[] = array_merge([
+            'timestamp' => now()->toIso8601String(),
+        ], $entry);
+
+        $run->update(['failures' => $failures]);
+    }
+
+    private function mapGraphFailureReasonCode(?int $status): string
+    {
+        return match (true) {
+            $status === 403 => 'graph_forbidden',
+            in_array($status, [429, 503], true) => 'graph_throttled',
+            in_array($status, [408, 500, 502, 504], true) => 'graph_transient',
+            default => 'unknown',
+        };
+    }
+
+    /**
+     * @return array{0:array{created:int,restored:int,failures:array<int,array{foundation_type:string,reason:string,status:int|string|null,reason_code:string}>},1:array<int,array{foundation_type:string,reason:string,status:int|string|null,reason_code:string}>}
+     */
+    private function captureFoundations(
+        BulkOperationService $bulkOperationService,
+        FoundationSnapshotService $foundationSnapshots,
+        Tenant $tenant,
+        BackupSet $backupSet,
+    ): array {
+        $types = config('tenantpilot.foundation_types', []);
+        $created = 0;
+        $restored = 0;
+        $failures = [];
+
+        foreach ($types as $typeConfig) {
+            $foundationType = $typeConfig['type'] ?? null;
+
+            if (! is_string($foundationType) || $foundationType === '') {
+                continue;
+            }
+
+            $result = $foundationSnapshots->fetchAll($tenant, $foundationType);
+
+            foreach (($result['failures'] ?? []) as $failure) {
+                if (! is_array($failure)) {
+                    continue;
+                }
+
+                $status = isset($failure['status']) && is_numeric($failure['status']) ? (int) $failure['status'] : null;
+                $reasonCode = $this->mapGraphFailureReasonCode($status);
+                $reason = $bulkOperationService->sanitizeFailureReason((string) ($failure['reason'] ?? 'Foundation capture failed.'));
+
+                $failures[] = [
+                    'foundation_type' => $foundationType,
+                    'reason' => $reason,
+                    'status' => $status,
+                    'reason_code' => $reasonCode,
+                ];
+            }
+
+            foreach (($result['items'] ?? []) as $snapshot) {
+                if (! is_array($snapshot)) {
+                    continue;
+                }
+
+                $sourceId = $snapshot['source_id'] ?? null;
+
+                if (! is_string($sourceId) || $sourceId === '') {
+                    continue;
+                }
+
+                $existing = BackupItem::withTrashed()
+                    ->where('backup_set_id', $backupSet->getKey())
+                    ->where('policy_type', $foundationType)
+                    ->where('policy_identifier', $sourceId)
+                    ->first();
+
+                if ($existing) {
+                    if ($existing->trashed()) {
+                        $existing->restore();
+                        $restored++;
+                    }
+
+                    continue;
+                }
+
+                BackupItem::create([
+                    'tenant_id' => $tenant->getKey(),
+                    'backup_set_id' => $backupSet->getKey(),
+                    'policy_id' => null,
+                    'policy_identifier' => $sourceId,
+                    'policy_type' => $foundationType,
+                    'platform' => $typeConfig['platform'] ?? null,
+                    'payload' => $snapshot['payload'] ?? [],
+                    'metadata' => $snapshot['metadata'] ?? [],
+                ]);
+
+                $created++;
+            }
+        }
+
+        return [
+            [
+                'created' => $created,
+                'restored' => $restored,
+                'failures' => $failures,
+            ],
+            $failures,
+        ];
+    }
+}

app/Livewire/BackupSetPolicyPickerTable.php

@@ -2,10 +2,15 @@
 
 namespace App\Livewire;
 
+use App\Filament\Resources\BulkOperationRunResource;
+use App\Jobs\AddPoliciesToBackupSetJob;
 use App\Models\BackupSet;
+use App\Models\BulkOperationRun;
 use App\Models\Policy;
 use App\Models\Tenant;
-use App\Services\Intune\BackupService;
+use App\Models\User;
+use App\Services\BulkOperationService;
+use App\Support\RunIdempotency;
 use Filament\Actions\BulkAction;
 use Filament\Notifications\Notification;
 use Filament\Tables\Columns\TextColumn;
@@ -15,6 +20,7 @@
 use Filament\Tables\TableComponent;
 use Illuminate\Contracts\View\View;
 use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\QueryException;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Str;
 
@@ -171,14 +177,82 @@ public function table(Table $table): Table
                 BulkAction::make('add_selected_to_backup_set')
                     ->label('Add selected')
                     ->icon('heroicon-m-plus')
-                    ->action(function (Collection $records, BackupService $service): void {
+                    ->authorize(function (): bool {
+                        $user = auth()->user();
+
+                        if (! $user instanceof User) {
+                            return false;
+                        }
+
+                        try {
+                            $tenant = Tenant::current();
+                        } catch (\RuntimeException) {
+                            return false;
+                        }
+
+                        if (! $user->canSyncTenant($tenant)) {
+                            return false;
+                        }
+
+                        return BackupSet::query()
+                            ->whereKey($this->backupSetId)
+                            ->where('tenant_id', $tenant->getKey())
+                            ->exists();
+                    })
+                    ->action(function (Collection $records, BulkOperationService $bulkOperationService): void {
                         $backupSet = BackupSet::query()->findOrFail($this->backupSetId);
-                        $tenant = $backupSet->tenant ?? Tenant::current();
+                        $tenant = null;
 
-                        $beforeFailures = (array) (($backupSet->metadata ?? [])['failures'] ?? []);
+                        try {
-                        $beforeFailureCount = count($beforeFailures);
+                            $tenant = Tenant::current();
+                        } catch (\RuntimeException) {
+                            $tenant = $backupSet->tenant;
+                        }
+                        $user = auth()->user();
 
-                        $policyIds = $records->pluck('id')->all();
+                        if (! $user instanceof User) {
+                            Notification::make()
+                                ->title('Not allowed')
+                                ->danger()
+                                ->send();
+
+                            return;
+                        }
+
+                        if (! $tenant instanceof Tenant) {
+                            Notification::make()
+                                ->title('Not allowed')
+                                ->danger()
+                                ->send();
+
+                            return;
+                        }
+
+                        if ((int) $tenant->getKey() !== (int) $backupSet->tenant_id) {
+                            Notification::make()
+                                ->title('Not allowed')
+                                ->danger()
+                                ->send();
+
+                            return;
+                        }
+
+                        if (! $user->canSyncTenant($tenant)) {
+                            Notification::make()
+                                ->title('Not allowed')
+                                ->danger()
+                                ->send();
+
+                            return;
+                        }
+
+                        $policyIds = $records
+                            ->pluck('id')
+                            ->map(fn (mixed $value): int => (int) $value)
+                            ->filter(fn (int $value): bool => $value > 0)
+                            ->unique()
+                            ->values()
+                            ->all();
 
                         if ($policyIds === []) {
                             Notification::make()
@@ -189,38 +263,109 @@ public function table(Table $table): Table
                             return;
                         }
 
-                        $service->addPoliciesToSet(
+                        sort($policyIds);
+
+                        $idempotencyKey = RunIdempotency::buildKey(
+                            tenantId: (int) $tenant->getKey(),
+                            operationType: 'backup_set.add_policies',
+                            targetId: (string) $backupSet->getKey(),
+                            context: [
+                                'policy_ids' => $policyIds,
+                                'include_assignments' => (bool) $this->include_assignments,
+                                'include_scope_tags' => (bool) $this->include_scope_tags,
+                                'include_foundations' => (bool) $this->include_foundations,
+                            ],
+                        );
+
+                        $existingRun = RunIdempotency::findActiveBulkOperationRun(
+                            tenantId: (int) $tenant->getKey(),
+                            idempotencyKey: $idempotencyKey,
+                        );
+
+                        if ($existingRun instanceof BulkOperationRun) {
+                            Notification::make()
+                                ->title('Add policies already queued')
+                                ->body('A matching run is already queued or running. Open the run to monitor progress.')
+                                ->actions([
+                                    \Filament\Actions\Action::make('view_run')
+                                        ->label('View run')
+                                        ->url(BulkOperationRunResource::getUrl('view', ['record' => $existingRun], tenant: $tenant)),
+                                ])
+                                ->info()
+                                ->send();
+
+                            return;
+                        }
+
+                        $selectionPayload = [
+                            'backup_set_id' => (int) $backupSet->getKey(),
+                            'policy_ids' => $policyIds,
+                            'options' => [
+                                'include_assignments' => (bool) $this->include_assignments,
+                                'include_scope_tags' => (bool) $this->include_scope_tags,
+                                'include_foundations' => (bool) $this->include_foundations,
+                            ],
+                        ];
+
+                        try {
+                            $run = $bulkOperationService->createRun(
                                 tenant: $tenant,
-                            backupSet: $backupSet,
+                                user: $user,
-                            policyIds: $policyIds,
+                                resource: 'backup_set',
-                            actorEmail: auth()->user()?->email,
+                                action: 'add_policies',
-                            actorName: auth()->user()?->name,
+                                itemIds: $selectionPayload,
-                            includeAssignments: $this->include_assignments,
+                                totalItems: count($policyIds),
-                            includeScopeTags: $this->include_scope_tags,
+                                idempotencyKey: $idempotencyKey,
-                            includeFoundations: $this->include_foundations,
+                            );
+                        } catch (QueryException $exception) {
+                            if ((string) $exception->getCode() === '23505') {
+                                $existingRun = RunIdempotency::findActiveBulkOperationRun(
+                                    tenantId: (int) $tenant->getKey(),
+                                    idempotencyKey: $idempotencyKey,
+                                );
+
+                                if ($existingRun instanceof BulkOperationRun) {
+                                    Notification::make()
+                                        ->title('Add policies already queued')
+                                        ->body('A matching run is already queued or running. Open the run to monitor progress.')
+                                        ->actions([
+                                            \Filament\Actions\Action::make('view_run')
+                                                ->label('View run')
+                                                ->url(BulkOperationRunResource::getUrl('view', ['record' => $existingRun], tenant: $tenant)),
+                                        ])
+                                        ->info()
+                                        ->send();
+
+                                    return;
+                                }
+                            }
+
+                            throw $exception;
+                        }
+
+                        AddPoliciesToBackupSetJob::dispatch(
+                            bulkRunId: (int) $run->getKey(),
+                            backupSetId: (int) $backupSet->getKey(),
+                            includeAssignments: (bool) $this->include_assignments,
+                            includeScopeTags: (bool) $this->include_scope_tags,
+                            includeFoundations: (bool) $this->include_foundations,
                         );
 
                         $notificationTitle = $this->include_foundations
-                            ? 'Backup items added'
+                            ? 'Backup items queued'
-                            : 'Policies added to backup';
+                            : 'Policies queued';
 
-                        $backupSet->refresh();
-
-                        $afterFailures = (array) (($backupSet->metadata ?? [])['failures'] ?? []);
-                        $afterFailureCount = count($afterFailures);
-
-                        if ($afterFailureCount > $beforeFailureCount) {
-                            Notification::make()
-                                ->title($notificationTitle.' with failures')
-                                ->body('Some policies could not be captured from Microsoft Graph. Check the backup set failures list for details.')
-                                ->warning()
-                                ->send();
-                        } else {
                         Notification::make()
                             ->title($notificationTitle)
+                            ->body('A background job has been queued. You can monitor progress in the run details or progress widget.')
+                            ->actions([
+                                \Filament\Actions\Action::make('view_run')
+                                    ->label('View run')
+                                    ->url(BulkOperationRunResource::getUrl('view', ['record' => $run], tenant: $tenant)),
+                            ])
                             ->success()
+                            ->sendToDatabase($user)
                             ->send();
-                        }
 
                         $this->resetTable();
                     }),

app/Services/BulkOperationService.php

@@ -43,15 +43,21 @@ public function createRun(
         string $resource,
         string $action,
         array $itemIds,
-        int $totalItems
+        int $totalItems,
+        ?string $idempotencyKey = null
     ): BulkOperationRun {
+        $effectiveTotalItems = $totalItems;
+
+        if (array_is_list($itemIds)) {
             $effectiveTotalItems = max($totalItems, count($itemIds));
+        }
 
         $run = BulkOperationRun::create([
             'tenant_id' => $tenant->id,
             'user_id' => $user->id,
             'resource' => $resource,
             'action' => $action,
+            'idempotency_key' => $idempotencyKey,
             'status' => 'pending',
             'item_ids' => $itemIds,
             'total_items' => $effectiveTotalItems,
@@ -94,17 +100,23 @@ public function recordSuccess(BulkOperationRun $run): void
         $run->increment('succeeded');
     }
 
-    public function recordFailure(BulkOperationRun $run, string $itemId, string $reason): void
+    public function recordFailure(BulkOperationRun $run, string $itemId, string $reason, ?string $reasonCode = null): void
     {
         $reason = $this->sanitizeFailureReason($reason);
 
         $failures = $run->failures ?? [];
-        $failures[] = [
+        $failureEntry = [
             'item_id' => $itemId,
             'reason' => $reason,
             'timestamp' => now()->toIso8601String(),
         ];
 
+        if (is_string($reasonCode) && $reasonCode !== '') {
+            $failureEntry['reason_code'] = $reasonCode;
+        }
+
+        $failures[] = $failureEntry;
+
         $run->update([
             'failures' => $failures,
             'processed_items' => $run->processed_items + 1,
@@ -118,18 +130,24 @@ public function recordSkipped(BulkOperationRun $run): void
         $run->increment('skipped');
     }
 
-    public function recordSkippedWithReason(BulkOperationRun $run, string $itemId, string $reason): void
+    public function recordSkippedWithReason(BulkOperationRun $run, string $itemId, string $reason, ?string $reasonCode = null): void
     {
         $reason = $this->sanitizeFailureReason($reason);
 
         $failures = $run->failures ?? [];
-        $failures[] = [
+        $failureEntry = [
             'item_id' => $itemId,
             'reason' => $reason,
             'type' => 'skipped',
             'timestamp' => now()->toIso8601String(),
         ];
 
+        if (is_string($reasonCode) && $reasonCode !== '') {
+            $failureEntry['reason_code'] = $reasonCode;
+        }
+
+        $failures[] = $failureEntry;
+
         $run->update([
             'failures' => $failures,
             'processed_items' => $run->processed_items + 1,

tests/Feature/BackupSets/AddPoliciesToBackupSetJobTest.php

@@ -0,0 +1,146 @@
+<?php
+
+use App\Jobs\AddPoliciesToBackupSetJob;
+use App\Models\BackupItem;
+use App\Models\BackupSet;
+use App\Models\BulkOperationRun;
+use App\Models\Policy;
+use App\Models\PolicyVersion;
+use App\Services\BulkOperationService;
+use App\Services\Intune\FoundationSnapshotService;
+use App\Services\Intune\PolicyCaptureOrchestrator;
+use App\Services\Intune\SnapshotValidator;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Mockery\MockInterface;
+
+uses(RefreshDatabase::class);
+
+it('records stable failure reason codes and keeps run counts consistent', function () {
+    [$user, $tenant] = createUserWithTenant(role: 'owner');
+    $this->actingAs($user);
+
+    $backupSet = BackupSet::factory()->create([
+        'tenant_id' => $tenant->id,
+        'name' => 'Test backup',
+        'status' => 'completed',
+        'metadata' => ['failures' => []],
+    ]);
+
+    $policyA = Policy::factory()->create([
+        'tenant_id' => $tenant->id,
+        'ignored_at' => null,
+    ]);
+
+    $policyB = Policy::factory()->create([
+        'tenant_id' => $tenant->id,
+        'ignored_at' => null,
+    ]);
+
+    $versionA = PolicyVersion::factory()->create([
+        'tenant_id' => $tenant->id,
+        'policy_id' => $policyA->id,
+        'policy_type' => $policyA->policy_type,
+        'platform' => $policyA->platform,
+        'snapshot' => ['id' => $policyA->external_id],
+    ]);
+
+    $run = BulkOperationRun::factory()->create([
+        'tenant_id' => $tenant->id,
+        'user_id' => $user->id,
+        'resource' => 'backup_set',
+        'action' => 'add_policies',
+        'status' => 'pending',
+        'total_items' => 2,
+        'item_ids' => [
+            'backup_set_id' => $backupSet->id,
+            'policy_ids' => [$policyA->id, $policyB->id],
+            'options' => [
+                'include_assignments' => true,
+                'include_scope_tags' => true,
+                'include_foundations' => false,
+            ],
+        ],
+        'failures' => [],
+    ]);
+
+    $this->mock(PolicyCaptureOrchestrator::class, function (MockInterface $mock) use ($policyA, $policyB, $tenant, $versionA) {
+        $mock->shouldReceive('capture')
+            ->twice()
+            ->andReturnUsing(function (
+                Policy $policy,
+                \App\Models\Tenant $tenantArg,
+                bool $includeAssignments = false,
+                bool $includeScopeTags = false,
+                ?string $createdBy = null,
+                array $metadata = []
+            ) use ($policyA, $policyB, $tenant, $versionA) {
+                expect($tenantArg->id)->toBe($tenant->id);
+                expect($includeAssignments)->toBeTrue();
+                expect($includeScopeTags)->toBeTrue();
+                expect($metadata['backup_set_id'] ?? null)->not->toBeNull();
+
+                if ($policy->is($policyA)) {
+                    return [
+                        'version' => $versionA,
+                        'captured' => [
+                            'payload' => [
+                                'id' => $policyA->external_id,
+                                '@odata.type' => '#microsoft.graph.deviceManagementConfigurationPolicy',
+                            ],
+                            'assignments' => [],
+                            'scope_tags' => ['ids' => ['0'], 'names' => ['Default']],
+                            'metadata' => [],
+                        ],
+                    ];
+                }
+
+                expect($policy->is($policyB))->toBeTrue();
+
+                return [
+                    'failure' => [
+                        'policy_id' => $policyB->id,
+                        'reason' => 'Forbidden',
+                        'status' => 403,
+                    ],
+                ];
+            });
+    });
+
+    $job = new AddPoliciesToBackupSetJob(
+        bulkRunId: (int) $run->getKey(),
+        backupSetId: (int) $backupSet->getKey(),
+        includeAssignments: true,
+        includeScopeTags: true,
+        includeFoundations: false,
+    );
+
+    $job->handle(
+        bulkOperationService: app(BulkOperationService::class),
+        captureOrchestrator: app(PolicyCaptureOrchestrator::class),
+        foundationSnapshots: $this->mock(FoundationSnapshotService::class),
+        snapshotValidator: app(SnapshotValidator::class),
+    );
+
+    $run->refresh();
+    $backupSet->refresh();
+
+    expect($run->status)->toBe('completed_with_errors');
+    expect($run->total_items)->toBe(2);
+    expect($run->processed_items)->toBe(2);
+    expect($run->succeeded)->toBe(1);
+    expect($run->failed)->toBe(1);
+    expect($run->skipped)->toBe(0);
+
+    expect(BackupItem::query()
+        ->where('backup_set_id', $backupSet->id)
+        ->where('policy_id', $policyA->id)
+        ->exists())->toBeTrue();
+
+    $failureEntry = collect($run->failures ?? [])
+        ->firstWhere('item_id', (string) $policyB->id);
+
+    expect($failureEntry)->not->toBeNull();
+    expect($failureEntry['reason_code'] ?? null)->toBe('graph_forbidden');
+
+    expect($backupSet->status)->toBe('partial');
+});

tests/Feature/Filament/BackupSetPolicyPickerTableTest.php

@@ -1,23 +1,31 @@
 <?php
 
+use App\Jobs\AddPoliciesToBackupSetJob;
 use App\Livewire\BackupSetPolicyPickerTable;
 use App\Models\BackupSet;
+use App\Models\BulkOperationRun;
 use App\Models\Policy;
 use App\Models\PolicyVersion;
 use App\Models\Tenant;
 use App\Models\User;
 use App\Services\Intune\BackupService;
+use App\Support\RunIdempotency;
+use Filament\Facades\Filament;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Queue;
 use Livewire\Livewire;
 use Mockery\MockInterface;
 
 uses(RefreshDatabase::class);
 
-test('policy picker table bulk adds selected policies to backup set', function () {
+test('policy picker table queues add policies job and creates a run (no inline capture)', function () {
-    $tenant = Tenant::factory()->create();
+    Queue::fake();
-    $tenant->makeCurrent();
 
-    $user = User::factory()->create();
+    [$user, $tenant] = createUserWithTenant(role: 'owner');
+    $this->actingAs($user);
+
+    $tenant->makeCurrent();
+    Filament::setTenant($tenant, true);
 
     $backupSet = BackupSet::factory()->create([
         'tenant_id' => $tenant->id,
@@ -30,22 +38,11 @@
         'last_synced_at' => now(),
     ]);
 
-    $this->mock(BackupService::class, function (MockInterface $mock) use ($tenant, $backupSet, $policies, $user) {
+    $this->mock(BackupService::class, function (MockInterface $mock) {
-        $mock->shouldReceive('addPoliciesToSet')
+        $mock->shouldReceive('addPoliciesToSet')->never();
-            ->once()
+    });
-            ->withArgs(function ($tenantArg, $backupSetArg, $policyIds, $actorEmail, $actorName, $includeAssignments, $includeScopeTags, $includeFoundations) use ($tenant, $backupSet, $policies, $user) {
-                expect($tenantArg->id)->toBe($tenant->id);
-                expect($backupSetArg->id)->toBe($backupSet->id);
-                expect($policyIds)->toBe($policies->pluck('id')->all());
-                expect($actorEmail)->toBe($user->email);
-                expect($actorName)->toBe($user->name);
-                expect($includeAssignments)->toBeTrue();
-                expect($includeScopeTags)->toBeTrue();
-                expect($includeFoundations)->toBeTrue();
 
-                return true;
+    bindFailHardGraphClient();
-            });
-    });
 
     Livewire::actingAs($user)
         ->test(BackupSetPolicyPickerTable::class, [
@@ -54,28 +51,119 @@
         ->callTableBulkAction('add_selected_to_backup_set', $policies)
         ->assertHasNoTableBulkActionErrors();
 
+    Queue::assertPushed(AddPoliciesToBackupSetJob::class, 1);
+
+    $policyIds = $policies
+        ->pluck('id')
+        ->map(fn (mixed $value): int => (int) $value)
+        ->sort()
+        ->values()
+        ->all();
+
+    $key = RunIdempotency::buildKey(
+        tenantId: (int) $tenant->getKey(),
+        operationType: 'backup_set.add_policies',
+        targetId: (string) $backupSet->getKey(),
+        context: [
+            'policy_ids' => $policyIds,
+            'include_assignments' => true,
+            'include_scope_tags' => true,
+            'include_foundations' => true,
+        ],
+    );
+
+    $run = BulkOperationRun::query()
+        ->where('tenant_id', $tenant->id)
+        ->where('resource', 'backup_set')
+        ->where('action', 'add_policies')
+        ->where('idempotency_key', $key)
+        ->latest('id')
+        ->first();
+
+    expect($run)->not->toBeNull();
+    expect($run?->status)->toBe('pending');
+    expect($run?->total_items)->toBe(count($policyIds));
+    expect($run?->item_ids['backup_set_id'] ?? null)->toBe($backupSet->getKey());
+    expect($run?->item_ids['policy_ids'] ?? null)->toBe($policyIds);
+    expect($run?->item_ids['options']['include_foundations'] ?? null)->toBeTrue();
+
     $notifications = session('filament.notifications', []);
 
     expect($notifications)->not->toBeEmpty();
-    expect(collect($notifications)->last()['title'] ?? null)->toBe('Backup items added');
+    expect(collect($notifications)->last()['title'] ?? null)->toBe('Backup items queued');
-    expect(collect($notifications)->last()['status'] ?? null)->toBe('success');
 });
 
-test('policy picker table does not warn if failures already existed but did not increase', function () {
+test('policy picker table reuses an active run on double click (idempotency)', function () {
-    $tenant = Tenant::factory()->create();
+    Queue::fake();
-    $tenant->makeCurrent();
 
-    $user = User::factory()->create();
+    [$user, $tenant] = createUserWithTenant(role: 'owner');
+    $this->actingAs($user);
+
+    $tenant->makeCurrent();
+    Filament::setTenant($tenant, true);
 
     $backupSet = BackupSet::factory()->create([
         'tenant_id' => $tenant->id,
         'name' => 'Test backup',
-        'status' => 'partial',
+    ]);
-        'metadata' => [
+
-            'failures' => [
+    $policies = Policy::factory()->count(2)->create([
-                ['policy_id' => 1, 'reason' => 'Previous failure', 'status' => 500],
+        'tenant_id' => $tenant->id,
-            ],
+        'ignored_at' => null,
+        'last_synced_at' => now(),
+    ]);
+
+    $policyIds = $policies
+        ->pluck('id')
+        ->map(fn (mixed $value): int => (int) $value)
+        ->sort()
+        ->values()
+        ->all();
+
+    $key = RunIdempotency::buildKey(
+        tenantId: (int) $tenant->getKey(),
+        operationType: 'backup_set.add_policies',
+        targetId: (string) $backupSet->getKey(),
+        context: [
+            'policy_ids' => $policyIds,
+            'include_assignments' => true,
+            'include_scope_tags' => true,
+            'include_foundations' => true,
         ],
+    );
+
+    Livewire::actingAs($user)
+        ->test(BackupSetPolicyPickerTable::class, [
+            'backupSetId' => $backupSet->id,
+        ])
+        ->callTableBulkAction('add_selected_to_backup_set', $policies);
+
+    Livewire::actingAs($user)
+        ->test(BackupSetPolicyPickerTable::class, [
+            'backupSetId' => $backupSet->id,
+        ])
+        ->callTableBulkAction('add_selected_to_backup_set', $policies);
+
+    expect(BulkOperationRun::query()
+        ->where('tenant_id', $tenant->id)
+        ->where('idempotency_key', $key)
+        ->count())->toBe(1);
+
+    Queue::assertPushed(AddPoliciesToBackupSetJob::class, 1);
+});
+
+test('policy picker table forbids readonly users from starting add policies (403)', function () {
+    Queue::fake();
+
+    [$user, $tenant] = createUserWithTenant(role: 'readonly');
+    $this->actingAs($user);
+
+    $tenant->makeCurrent();
+    Filament::setTenant($tenant, true);
+
+    $backupSet = BackupSet::factory()->create([
+        'tenant_id' => $tenant->id,
+        'name' => 'Test backup',
     ]);
 
     $policies = Policy::factory()->count(1)->create([
@@ -84,74 +172,70 @@
         'last_synced_at' => now(),
     ]);
 
-    $this->mock(BackupService::class, function (MockInterface $mock) use ($backupSet) {
+    $thrown = null;
-        $mock->shouldReceive('addPoliciesToSet')
-            ->once()
-            ->andReturn($backupSet);
-    });
 
+    try {
         Livewire::actingAs($user)
             ->test(BackupSetPolicyPickerTable::class, [
                 'backupSetId' => $backupSet->id,
             ])
-        ->callTableBulkAction('add_selected_to_backup_set', $policies)
+            ->callTableBulkAction('add_selected_to_backup_set', $policies);
-        ->assertHasNoTableBulkActionErrors();
+    } catch (Throwable $exception) {
+        $thrown = $exception;
+    }
 
-    $notifications = session('filament.notifications', []);
+    expect($thrown)->not->toBeNull();
 
-    expect($notifications)->not->toBeEmpty();
+    Queue::assertNothingPushed();
-    expect(collect($notifications)->last()['title'] ?? null)->toBe('Backup items added');
+
-    expect(collect($notifications)->last()['status'] ?? null)->toBe('success');
+    expect(BulkOperationRun::query()->where('tenant_id', $tenant->id)->exists())->toBeFalse();
 });
 
-test('policy picker table warns when new failures were added', function () {
+test('policy picker table rejects cross-tenant starts (403) with no run records created', function () {
-    $tenant = Tenant::factory()->create();
+    Queue::fake();
-    $tenant->makeCurrent();
+
+    $tenantA = Tenant::factory()->create();
+    $tenantB = Tenant::factory()->create();
 
     $user = User::factory()->create();
-
+    $user->tenants()->syncWithoutDetaching([
-    $backupSet = BackupSet::factory()->create([
+        $tenantA->getKey() => ['role' => 'owner'],
-        'tenant_id' => $tenant->id,
+        $tenantB->getKey() => ['role' => 'owner'],
-        'name' => 'Test backup',
-        'status' => 'completed',
-        'metadata' => ['failures' => []],
     ]);
 
-    $policies = Policy::factory()->count(1)->create([
+    $this->actingAs($user);
-        'tenant_id' => $tenant->id,
+
+    $tenantA->makeCurrent();
+    Filament::setTenant($tenantA, true);
+
+    $backupSetB = BackupSet::factory()->create([
+        'tenant_id' => $tenantB->id,
+        'name' => 'Tenant B backup',
+    ]);
+
+    $policiesB = Policy::factory()->count(1)->create([
+        'tenant_id' => $tenantB->id,
         'ignored_at' => null,
         'last_synced_at' => now(),
     ]);
 
-    $this->mock(BackupService::class, function (MockInterface $mock) use ($backupSet) {
+    $thrown = null;
-        $mock->shouldReceive('addPoliciesToSet')
-            ->once()
-            ->andReturnUsing(function () use ($backupSet) {
-                $backupSet->update([
-                    'status' => 'partial',
-                    'metadata' => [
-                        'failures' => [
-                            ['policy_id' => 123, 'reason' => 'New failure', 'status' => 500],
-                        ],
-                    ],
-                ]);
-
-                return $backupSet->refresh();
-            });
-    });
 
+    try {
         Livewire::actingAs($user)
             ->test(BackupSetPolicyPickerTable::class, [
-            'backupSetId' => $backupSet->id,
+                'backupSetId' => $backupSetB->id,
             ])
-        ->callTableBulkAction('add_selected_to_backup_set', $policies)
+            ->callTableBulkAction('add_selected_to_backup_set', $policiesB);
-        ->assertHasNoTableBulkActionErrors();
+    } catch (Throwable $exception) {
+        $thrown = $exception;
+    }
 
-    $notifications = session('filament.notifications', []);
+    expect($thrown)->not->toBeNull();
 
-    expect($notifications)->not->toBeEmpty();
+    Queue::assertNothingPushed();
-    expect(collect($notifications)->last()['title'] ?? null)->toBe('Backup items added with failures');
+
-    expect(collect($notifications)->last()['status'] ?? null)->toBe('warning');
+    expect(BulkOperationRun::query()->where('tenant_id', $tenantB->id)->exists())->toBeFalse();
 });
 
 test('policy picker table can filter by has versions', function () {