Implements Spec 096 ops polish bundle: - Persist durable OperationRun.summary_counts for assignment fetch/restore (final attempt wins) - Server-side dedupe for assignment jobs (15-minute cooldown + non-canonical skip) - Track ReconcileAdapterRunsJob via workspace-scoped OperationRun + stable failure codes + overlap prevention - Seed DX: ensure seeded tenants use UUID v4 external_id and seed satisfies workspace_id NOT NULL constraints Verification (local / evidence-based): - `vendor/bin/sail artisan test --compact tests/Feature/Operations/AssignmentRunSummaryCountsTest.php tests/Feature/Operations/AssignmentJobDedupeTest.php tests/Feature/Operations/ReconcileAdapterRunsJobTrackingTest.php tests/Feature/Seed/PoliciesSeederExternalIdTest.php` - `vendor/bin/sail bin pint --dirty` Spec artifacts included under `specs/096-ops-polish-assignment-dedupe-system-tracking/` (spec/plan/tasks/checklists). Co-authored-by: Ahmed Darrazi <ahmed.darrazi@live.de> Reviewed-on: #115
417 lines
15 KiB
PHP
417 lines
15 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Models\OperationRun;
|
|
use App\Models\RestoreRun;
|
|
use App\Models\Tenant;
|
|
use App\Models\User;
|
|
use App\Services\AssignmentRestoreService;
|
|
use App\Services\OperationRunService;
|
|
use App\Support\OperationRunOutcome;
|
|
use App\Support\OperationRunStatus;
|
|
use App\Support\OpsUx\AssignmentJobFingerprint;
|
|
use App\Support\OpsUx\RunFailureSanitizer;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Illuminate\Support\Facades\Log;
|
|
use RuntimeException;
|
|
|
|
class RestoreAssignmentsJob implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
private const string OPERATION_TYPE = 'assignments.restore';
|
|
|
|
private const int DEDUPE_COOLDOWN_MINUTES = 15;
|
|
|
|
private const int EXECUTION_LOCK_TTL_SECONDS = 900;
|
|
|
|
public ?OperationRun $operationRun = null;
|
|
|
|
public int $tries = 1;
|
|
|
|
public int $backoff = 0;
|
|
|
|
/**
|
|
* Create a new job instance.
|
|
*/
|
|
public function __construct(
|
|
public int $restoreRunId,
|
|
public int $tenantId,
|
|
public string $policyType,
|
|
public string $policyId,
|
|
public array $assignments,
|
|
public array $groupMapping,
|
|
public array $foundationMapping = [],
|
|
public ?string $actorEmail = null,
|
|
public ?string $actorName = null,
|
|
?OperationRun $operationRun = null,
|
|
) {
|
|
$this->operationRun = $operationRun;
|
|
}
|
|
|
|
public static function dispatchTracked(
|
|
int $restoreRunId,
|
|
Tenant $tenant,
|
|
string $policyType,
|
|
string $policyId,
|
|
array $assignments,
|
|
array $groupMapping,
|
|
array $foundationMapping = [],
|
|
?string $actorEmail = null,
|
|
?string $actorName = null,
|
|
?User $initiator = null,
|
|
): OperationRun {
|
|
/** @var OperationRunService $operationRunService */
|
|
$operationRunService = app(OperationRunService::class);
|
|
|
|
$identityInputs = self::operationRunIdentityInputs(
|
|
restoreRunId: $restoreRunId,
|
|
tenantId: (int) $tenant->getKey(),
|
|
policyType: $policyType,
|
|
policyId: $policyId,
|
|
assignments: $assignments,
|
|
groupMapping: $groupMapping,
|
|
foundationMapping: $foundationMapping,
|
|
);
|
|
|
|
$run = $operationRunService->ensureRunWithIdentityCooldown(
|
|
tenant: $tenant,
|
|
type: self::OPERATION_TYPE,
|
|
identityInputs: $identityInputs,
|
|
context: self::operationRunContext(
|
|
restoreRunId: $restoreRunId,
|
|
policyType: $policyType,
|
|
policyId: $policyId,
|
|
assignments: $assignments,
|
|
),
|
|
cooldownMinutes: self::DEDUPE_COOLDOWN_MINUTES,
|
|
initiator: $initiator,
|
|
);
|
|
|
|
if ($run->wasRecentlyCreated) {
|
|
dispatch(new self(
|
|
restoreRunId: $restoreRunId,
|
|
tenantId: (int) $tenant->getKey(),
|
|
policyType: $policyType,
|
|
policyId: $policyId,
|
|
assignments: $assignments,
|
|
groupMapping: $groupMapping,
|
|
foundationMapping: $foundationMapping,
|
|
actorEmail: $actorEmail,
|
|
actorName: $actorName,
|
|
operationRun: $run,
|
|
));
|
|
}
|
|
|
|
return $run;
|
|
}
|
|
|
|
/**
|
|
* Execute the job.
|
|
*
|
|
* @return array{outcomes: array<int, array<string, mixed>>, summary: array{success:int,failed:int,skipped:int}}
|
|
*/
|
|
public function handle(
|
|
AssignmentRestoreService $assignmentRestoreService,
|
|
OperationRunService $operationRunService,
|
|
): array {
|
|
$restoreRun = RestoreRun::query()->find($this->restoreRunId);
|
|
$tenant = Tenant::query()->find($this->tenantId);
|
|
|
|
if (! $tenant instanceof Tenant || ! $restoreRun instanceof RestoreRun) {
|
|
Log::warning('RestoreAssignmentsJob missing context', [
|
|
'restore_run_id' => $this->restoreRunId,
|
|
'tenant_id' => $this->tenantId,
|
|
]);
|
|
|
|
return [
|
|
'outcomes' => [],
|
|
'summary' => ['success' => 0, 'failed' => 0, 'skipped' => 0],
|
|
];
|
|
}
|
|
|
|
$identityInputs = self::operationRunIdentityInputs(
|
|
restoreRunId: $this->restoreRunId,
|
|
tenantId: $this->tenantId,
|
|
policyType: $this->policyType,
|
|
policyId: $this->policyId,
|
|
assignments: $this->assignments,
|
|
groupMapping: $this->groupMapping,
|
|
foundationMapping: $this->foundationMapping,
|
|
);
|
|
|
|
$fingerprint = AssignmentJobFingerprint::forRestore(
|
|
restoreRunId: $this->restoreRunId,
|
|
tenantId: $this->tenantId,
|
|
policyType: $this->policyType,
|
|
policyId: $this->policyId,
|
|
assignments: $this->assignments,
|
|
groupMapping: $this->groupMapping,
|
|
foundationMapping: $this->foundationMapping,
|
|
);
|
|
|
|
if (! $this->operationRun instanceof OperationRun) {
|
|
$this->operationRun = $operationRunService->ensureRunWithIdentityCooldown(
|
|
tenant: $tenant,
|
|
type: self::OPERATION_TYPE,
|
|
identityInputs: $identityInputs,
|
|
context: self::operationRunContext(
|
|
restoreRunId: $this->restoreRunId,
|
|
policyType: $this->policyType,
|
|
policyId: $this->policyId,
|
|
assignments: $this->assignments,
|
|
),
|
|
cooldownMinutes: self::DEDUPE_COOLDOWN_MINUTES,
|
|
);
|
|
}
|
|
|
|
$canonicalRun = $operationRunService->findCanonicalRunWithIdentity(
|
|
tenant: $tenant,
|
|
type: self::OPERATION_TYPE,
|
|
identityInputs: $identityInputs,
|
|
cooldownMinutes: self::DEDUPE_COOLDOWN_MINUTES,
|
|
);
|
|
|
|
if ($canonicalRun instanceof OperationRun && $canonicalRun->status === OperationRunStatus::Completed->value) {
|
|
Log::info('RestoreAssignmentsJob: Skipping because identity is in cooldown window', [
|
|
'restore_run_id' => $this->restoreRunId,
|
|
'operation_run_id' => (int) $canonicalRun->getKey(),
|
|
]);
|
|
|
|
return [
|
|
'outcomes' => [],
|
|
'summary' => ['success' => 0, 'failed' => 0, 'skipped' => 0],
|
|
];
|
|
}
|
|
|
|
if ($canonicalRun instanceof OperationRun
|
|
&& $this->operationRun instanceof OperationRun
|
|
&& (int) $canonicalRun->getKey() !== (int) $this->operationRun->getKey()
|
|
) {
|
|
Log::info('RestoreAssignmentsJob: Skipping non-canonical execution', [
|
|
'restore_run_id' => $this->restoreRunId,
|
|
'canonical_run_id' => (int) $canonicalRun->getKey(),
|
|
'job_run_id' => (int) $this->operationRun->getKey(),
|
|
]);
|
|
|
|
return [
|
|
'outcomes' => [],
|
|
'summary' => ['success' => 0, 'failed' => 0, 'skipped' => 0],
|
|
];
|
|
}
|
|
|
|
$run = $this->operationRun instanceof OperationRun ? $this->operationRun : $canonicalRun;
|
|
|
|
if (! $run instanceof OperationRun) {
|
|
throw new RuntimeException('OperationRun is required for RestoreAssignmentsJob execution.');
|
|
}
|
|
|
|
$executionIdentityKey = AssignmentJobFingerprint::executionIdentityKey(
|
|
jobType: self::OPERATION_TYPE,
|
|
tenantId: (int) $tenant->getKey(),
|
|
fingerprint: $fingerprint,
|
|
operationRunId: (int) $run->getKey(),
|
|
);
|
|
|
|
$lock = Cache::lock('assignment-job:'.$executionIdentityKey, self::EXECUTION_LOCK_TTL_SECONDS);
|
|
|
|
if (! $lock->get()) {
|
|
Log::info('RestoreAssignmentsJob: Overlap prevented for duplicate execution', [
|
|
'restore_run_id' => $this->restoreRunId,
|
|
'operation_run_id' => (int) $run->getKey(),
|
|
'execution_identity' => $executionIdentityKey,
|
|
]);
|
|
|
|
return [
|
|
'outcomes' => [],
|
|
'summary' => ['success' => 0, 'failed' => 0, 'skipped' => 0],
|
|
];
|
|
}
|
|
|
|
try {
|
|
if ($run->status !== OperationRunStatus::Completed->value) {
|
|
$operationRunService->updateRun($run, OperationRunStatus::Running->value);
|
|
}
|
|
|
|
$result = $assignmentRestoreService->restore(
|
|
tenant: $tenant,
|
|
policyType: $this->policyType,
|
|
policyId: $this->policyId,
|
|
assignments: $this->assignments,
|
|
groupMapping: $this->groupMapping,
|
|
foundationMapping: $this->foundationMapping,
|
|
restoreRun: $restoreRun,
|
|
actorEmail: $this->actorEmail,
|
|
actorName: $this->actorName,
|
|
);
|
|
|
|
$summary = is_array($result['summary'] ?? null) ? $result['summary'] : [];
|
|
$success = (int) ($summary['success'] ?? 0);
|
|
$failed = (int) ($summary['failed'] ?? 0);
|
|
$skipped = (int) ($summary['skipped'] ?? 0);
|
|
$total = $success + $failed + $skipped;
|
|
$processed = $success + $failed;
|
|
|
|
$outcome = OperationRunOutcome::Succeeded->value;
|
|
|
|
if ($failed > 0 && $success > 0) {
|
|
$outcome = OperationRunOutcome::PartiallySucceeded->value;
|
|
} elseif ($failed > 0) {
|
|
$outcome = OperationRunOutcome::Failed->value;
|
|
}
|
|
|
|
$failures = $this->extractFailures(
|
|
outcomes: is_array($result['outcomes'] ?? null) ? $result['outcomes'] : [],
|
|
fallbackMessage: 'Assignments restore failed.',
|
|
);
|
|
|
|
$operationRunService->updateRun(
|
|
$run,
|
|
status: OperationRunStatus::Completed->value,
|
|
outcome: $outcome,
|
|
summaryCounts: [
|
|
'total' => $total,
|
|
'processed' => $processed,
|
|
'failed' => $failed,
|
|
],
|
|
failures: $failures,
|
|
);
|
|
|
|
return [
|
|
'outcomes' => is_array($result['outcomes'] ?? null) ? $result['outcomes'] : [],
|
|
'summary' => [
|
|
'success' => $success,
|
|
'failed' => $failed,
|
|
'skipped' => $skipped,
|
|
],
|
|
];
|
|
} catch (\Throwable $throwable) {
|
|
Log::error('RestoreAssignmentsJob failed', [
|
|
'restore_run_id' => $this->restoreRunId,
|
|
'policy_id' => $this->policyId,
|
|
'operation_run_id' => (int) $run->getKey(),
|
|
'error' => $throwable->getMessage(),
|
|
]);
|
|
|
|
$safeMessage = RunFailureSanitizer::sanitizeMessage($throwable->getMessage());
|
|
|
|
$operationRunService->updateRun(
|
|
$run,
|
|
status: OperationRunStatus::Completed->value,
|
|
outcome: OperationRunOutcome::Failed->value,
|
|
summaryCounts: [
|
|
'total' => max(1, count($this->assignments)),
|
|
'processed' => 0,
|
|
'failed' => max(1, count($this->assignments)),
|
|
],
|
|
failures: [[
|
|
'code' => 'assignments.restore_failed',
|
|
'reason_code' => RunFailureSanitizer::normalizeReasonCode($throwable->getMessage()),
|
|
'message' => $safeMessage !== '' ? $safeMessage : 'Assignments restore failed.',
|
|
]],
|
|
);
|
|
|
|
return [
|
|
'outcomes' => [[
|
|
'status' => 'failed',
|
|
'reason' => $safeMessage !== '' ? $safeMessage : 'Assignments restore failed.',
|
|
]],
|
|
'summary' => ['success' => 0, 'failed' => 1, 'skipped' => 0],
|
|
];
|
|
} finally {
|
|
$lock->release();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param array<int, array<string, mixed>> $assignments
|
|
* @param array<string, mixed> $groupMapping
|
|
* @param array<string, mixed> $foundationMapping
|
|
* @return array<string, int|string>
|
|
*/
|
|
private static function operationRunIdentityInputs(
|
|
int $restoreRunId,
|
|
int $tenantId,
|
|
string $policyType,
|
|
string $policyId,
|
|
array $assignments,
|
|
array $groupMapping,
|
|
array $foundationMapping = [],
|
|
): array {
|
|
return [
|
|
'tenant_id' => $tenantId,
|
|
'job_type' => self::OPERATION_TYPE,
|
|
'fingerprint' => AssignmentJobFingerprint::forRestore(
|
|
restoreRunId: $restoreRunId,
|
|
tenantId: $tenantId,
|
|
policyType: $policyType,
|
|
policyId: $policyId,
|
|
assignments: $assignments,
|
|
groupMapping: $groupMapping,
|
|
foundationMapping: $foundationMapping,
|
|
),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param array<int, array<string, mixed>> $assignments
|
|
* @return array<string, int|string>
|
|
*/
|
|
private static function operationRunContext(
|
|
int $restoreRunId,
|
|
string $policyType,
|
|
string $policyId,
|
|
array $assignments,
|
|
): array {
|
|
return [
|
|
'restore_run_id' => $restoreRunId,
|
|
'policy_type' => trim($policyType),
|
|
'policy_id' => trim($policyId),
|
|
'assignment_item_count' => count($assignments),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param array<int, array<string, mixed>> $outcomes
|
|
* @return array<int, array{code:string, reason_code:string, message:string}>
|
|
*/
|
|
private function extractFailures(array $outcomes, string $fallbackMessage): array
|
|
{
|
|
$failures = [];
|
|
|
|
foreach ($outcomes as $outcome) {
|
|
if (! is_array($outcome)) {
|
|
continue;
|
|
}
|
|
|
|
if (($outcome['status'] ?? null) !== 'failed') {
|
|
continue;
|
|
}
|
|
|
|
$messageCandidate = is_string($outcome['reason'] ?? null)
|
|
? (string) $outcome['reason']
|
|
: (is_string($outcome['message'] ?? null) ? (string) $outcome['message'] : $fallbackMessage);
|
|
|
|
$message = RunFailureSanitizer::sanitizeMessage($messageCandidate);
|
|
|
|
$failures[] = [
|
|
'code' => 'assignments.restore_failed',
|
|
'reason_code' => RunFailureSanitizer::normalizeReasonCode($messageCandidate),
|
|
'message' => $message !== '' ? $message : $fallbackMessage,
|
|
];
|
|
}
|
|
|
|
if ($failures === []) {
|
|
return [];
|
|
}
|
|
|
|
return array_slice($failures, 0, 10);
|
|
}
|
|
}
|