feat: implement restore high risk operation reconciliation #435

Merged
ahmido merged 1 commits from 364-restore-high-risk-operation-reconciliation into platform-dev 2026-06-07 14:10:35 +00:00
23 changed files with 1801 additions and 117 deletions

View File

@ -1601,8 +1601,22 @@ public static function restoreContinuation(OperationRun $record): ?array
$restoreRunId = is_numeric($context['restore_run_id'] ?? null) ? (int) $context['restore_run_id'] : null;
$restoreRun = $restoreRunId !== null
? RestoreRun::query()->find($restoreRunId)
: RestoreRun::query()->where('operation_run_id', (int) $record->getKey())->latest('id')->first();
? RestoreRun::query()
->whereKey($restoreRunId)
->where('workspace_id', (int) $record->workspace_id)
->where('managed_environment_id', (int) $record->managed_environment_id)
->where(function (Builder $query) use ($record): void {
$query
->whereNull('operation_run_id')
->orWhere('operation_run_id', (int) $record->getKey());
})
->first()
: RestoreRun::query()
->where('workspace_id', (int) $record->workspace_id)
->where('managed_environment_id', (int) $record->managed_environment_id)
->where('operation_run_id', (int) $record->getKey())
->latest('id')
->first();
if (! $restoreRun instanceof RestoreRun) {
return null;

View File

@ -5,12 +5,14 @@
use App\Models\OperationRun;
use App\Models\RestoreRun;
use App\Services\OperationRunService;
use App\Support\Operations\Reconciliation\RestoreExecuteReconciliationAdapter;
use App\Support\RestoreRunStatus;
class SyncRestoreRunToOperationRun
{
public function __construct(
public OperationRunService $service
public OperationRunService $service,
public RestoreExecuteReconciliationAdapter $restoreExecuteAdapter,
) {}
public function handle(RestoreRun $restoreRun): void
@ -68,6 +70,23 @@ public function handle(RestoreRun $restoreRun): void
});
}
$this->syncRunContext($opRun, $inputs);
if ($this->isTerminalStatus($status)) {
$result = $this->restoreExecuteAdapter->reconcile($opRun->fresh() ?? $opRun);
if ($result?->shouldFinalizeRun()) {
$this->service->applyReconciliationResult(
run: $opRun->fresh() ?? $opRun,
result: $result,
source: 'restore_run_observer',
adapter: $this->restoreExecuteAdapter->key(),
);
}
return;
}
[$opStatus, $opOutcome, $failures] = $this->mapStatus($status);
$summaryCounts = [];
@ -94,17 +113,38 @@ public function handle(RestoreRun $restoreRun): void
protected function mapStatus(RestoreRunStatus $status): array
{
return match ($status) {
RestoreRunStatus::Previewed => ['completed', 'succeeded', []],
RestoreRunStatus::Pending => ['queued', 'pending', []],
RestoreRunStatus::Queued => ['queued', 'pending', []],
RestoreRunStatus::Running => ['running', 'pending', []],
RestoreRunStatus::Completed => ['completed', 'succeeded', []],
RestoreRunStatus::Partial, RestoreRunStatus::CompletedWithErrors => ['completed', 'partially_succeeded', []],
RestoreRunStatus::Failed, RestoreRunStatus::Aborted => ['completed', 'failed', []],
RestoreRunStatus::Cancelled => ['completed', 'failed', [
['code' => 'restore.cancelled', 'message' => 'Restore run was cancelled.'],
]],
default => ['running', 'pending', []],
};
}
/**
* @param array<string, mixed> $inputs
*/
private function syncRunContext(OperationRun $opRun, array $inputs): void
{
$context = is_array($opRun->context) ? $opRun->context : [];
$updatedContext = array_replace($context, $inputs);
if ($updatedContext === $context) {
return;
}
$opRun->forceFill(['context' => $updatedContext])->save();
}
private function isTerminalStatus(RestoreRunStatus $status): bool
{
return in_array($status, [
RestoreRunStatus::Previewed,
RestoreRunStatus::Completed,
RestoreRunStatus::Partial,
RestoreRunStatus::Failed,
RestoreRunStatus::Cancelled,
RestoreRunStatus::Aborted,
RestoreRunStatus::CompletedWithErrors,
], true);
}
}

View File

@ -2,13 +2,13 @@
namespace App\Support;
use App\Filament\Pages\InventoryCoverage as InventoryCoveragePage;
use App\Filament\Resources\BackupScheduleResource;
use App\Filament\Resources\BackupSetResource;
use App\Filament\Resources\BaselineSnapshotResource;
use App\Filament\Resources\EntraGroupResource;
use App\Filament\Resources\EnvironmentReviewResource;
use App\Filament\Resources\EvidenceSnapshotResource;
use App\Filament\Pages\InventoryCoverage as InventoryCoveragePage;
use App\Filament\Resources\InventoryItemResource;
use App\Filament\Resources\PolicyResource;
use App\Filament\Resources\RestoreRunResource;
@ -17,6 +17,7 @@
use App\Models\EvidenceSnapshot;
use App\Models\ManagedEnvironment;
use App\Models\OperationRun;
use App\Models\RestoreRun;
use App\Models\ReviewPack;
use App\Models\Workspace;
use App\Support\Navigation\CanonicalNavigationContext;
@ -232,7 +233,21 @@ public static function related(OperationRun $run, ?ManagedEnvironment $tenant):
$restoreRunId = $context['restore_run_id'] ?? null;
if (is_numeric($restoreRunId)) {
$links['Restore Run'] = RestoreRunResource::getUrl('view', ['record' => (int) $restoreRunId], tenant: $tenant);
$restoreRun = RestoreRun::query()
->whereKey((int) $restoreRunId)
->where('workspace_id', (int) $run->workspace_id)
->where('managed_environment_id', (int) $run->managed_environment_id)
->first();
if (
$restoreRun instanceof RestoreRun
&& (
! is_numeric($restoreRun->operation_run_id)
|| (int) $restoreRun->operation_run_id === (int) $run->getKey()
)
) {
$links['Restore Run'] = RestoreRunResource::getUrl('view', ['record' => $restoreRun], tenant: $tenant);
}
}
}

View File

@ -4,11 +4,13 @@
namespace App\Support\Operations\Reconciliation;
use App\Models\AuditLog;
use App\Models\EvidenceSnapshot;
use App\Models\OperationRun;
use App\Models\RestoreRun;
use App\Support\Evidence\EvidenceSnapshotStatus;
use App\Support\OperationRunOutcome;
use App\Support\OperationRunStatus;
use App\Support\Operations\LifecycleReconciliationReason;
use App\Support\RestoreRunStatus;
use Throwable;
@ -38,13 +40,16 @@ public function reconcile(OperationRun $run): ?ReconciliationResult
return null;
}
$restoreRun = RestoreRun::query()
->where('managed_environment_id', $run->managed_environment_id)
->whereKey((int) $restoreRunId)
->first();
$restoreRunId = (int) $restoreRunId;
$baseEvidence = $this->baseEvidence($run, $restoreRunId);
[$restoreRun, $scopeProblem] = $this->resolveRestoreRun($run, $restoreRunId);
if (! $restoreRun instanceof RestoreRun) {
return null;
if ($scopeProblem !== null) {
return ReconciliationResult::notReconciled(
reasonCode: $scopeProblem['reason_code'],
reasonMessage: $scopeProblem['message'],
evidence: $baseEvidence + $scopeProblem['evidence'],
);
}
$restoreStatus = RestoreRunStatus::fromString($restoreRun->status);
@ -53,24 +58,135 @@ public function reconcile(OperationRun $run): ?ReconciliationResult
return null;
}
[$status, $outcome, $failures] = $this->mapRestoreToOperationRun($restoreRun, $restoreStatus);
if (
is_numeric($restoreRun->operation_run_id)
&& (int) $restoreRun->operation_run_id !== (int) $run->getKey()
) {
return ReconciliationResult::notReconciled(
reasonCode: 'restore.scope_mismatch',
reasonMessage: 'The restore run is linked to a different operation run and cannot safely reconcile this record.',
evidence: $baseEvidence + [
'restore_operation_run_id' => (int) $restoreRun->operation_run_id,
'restore_status' => $restoreStatus?->value,
],
);
}
return new ReconciliationResult(
decision: 'reconciled_succeeded',
status: $status,
outcome: $outcome,
reasonCode: LifecycleReconciliationReason::AdapterOutOfSync->value,
reasonMessage: 'A related restore record reached terminal truth before the operation run was updated.',
evidence: array_filter([
'restore_run_id' => (int) $restoreRun->getKey(),
'restore_status' => $restoreStatus?->value,
'restore_started_at' => $restoreRun->started_at?->toIso8601String(),
'restore_completed_at' => $restoreRun->completed_at?->toIso8601String(),
], static fn (mixed $value): bool => $value !== null),
$summaryCounts = $this->buildSummaryCounts($restoreRun);
$proofCounts = $this->proofCounts($restoreRun);
$evidenceSnapshot = $this->currentEvidenceSnapshot($run);
$completionAudit = $this->completionAudit($run, $restoreRun);
$evidence = $baseEvidence + $this->restoreEvidence(
restoreRun: $restoreRun,
status: $restoreStatus,
proofCounts: $proofCounts,
evidenceSnapshot: $evidenceSnapshot,
completionAudit: $completionAudit,
);
if ($restoreStatus === RestoreRunStatus::Previewed || (bool) ($restoreRun->is_dry_run ?? false)) {
return ReconciliationResult::blocked(
reasonCode: 'restore.preview_only',
reasonMessage: 'The restore run only proves preview truth; no live restore execution was performed.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
$failureReason = $this->failureReason($restoreRun);
if ($restoreStatus === RestoreRunStatus::Cancelled) {
return ReconciliationResult::failedUnrecoverable(
reasonCode: 'restore.cancelled',
reasonMessage: 'Restore run was cancelled.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if (in_array($restoreStatus, [RestoreRunStatus::Failed, RestoreRunStatus::Aborted], true)) {
return ReconciliationResult::failedUnrecoverable(
reasonCode: 'restore.failed',
reasonMessage: $failureReason !== '' ? $failureReason : 'Restore failed.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if ($restoreRun->started_at === null || $restoreRun->completed_at === null) {
return ReconciliationResult::failedUnrecoverable(
reasonCode: 'restore.execution_proof_missing',
reasonMessage: 'The restore reached a terminal state without complete execution timestamps.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if ($failureReason !== '') {
return ReconciliationResult::failedUnrecoverable(
reasonCode: 'restore.provider_rejected',
reasonMessage: $failureReason,
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if (in_array($restoreStatus, [RestoreRunStatus::Partial, RestoreRunStatus::CompletedWithErrors], true)) {
return $this->partialResult(
reasonCode: 'restore.results_mixed',
reasonMessage: 'The restore reached a terminal state with partial or warning results.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if (! $proofCounts['has_result_truth']) {
return ReconciliationResult::failedUnrecoverable(
reasonCode: 'restore.results_missing',
reasonMessage: 'The restore completed without repo-backed item or aggregate result proof.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if (
$proofCounts['failed'] > 0
|| $proofCounts['skipped'] > 0
|| $proofCounts['partial'] > 0
|| $proofCounts['non_applied'] > 0
) {
return $this->partialResult(
reasonCode: 'restore.results_mixed',
reasonMessage: 'The restore completed with failed, skipped, partial, or non-applied work.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if (! $evidenceSnapshot instanceof EvidenceSnapshot && ! $this->hasExplicitVerificationProof($restoreRun)) {
return $this->partialResult(
reasonCode: 'restore.verification_required',
reasonMessage: 'The restore completed, but no current post-run evidence snapshot proves recovery.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
if (! $completionAudit instanceof AuditLog) {
return $this->partialResult(
reasonCode: 'restore.audit_missing',
reasonMessage: 'The restore completed, but completion audit continuity is missing.',
evidence: $evidence,
summaryCounts: $summaryCounts,
);
}
return ReconciliationResult::reconciledSucceeded(
reasonCode: 'restore.proof_complete',
reasonMessage: 'The restore completed with result proof, post-run evidence, and audit continuity.',
evidence: $evidence,
related: [],
summaryCounts: $this->buildSummaryCounts($restoreRun),
failures: $failures,
safeForAutoCompletion: true,
summaryCounts: $summaryCounts,
);
}
@ -79,6 +195,49 @@ public function reconcileException(OperationRun $run, Throwable $throwable): ?Re
return null;
}
/**
* @return array{0:RestoreRun|null,1:array{reason_code:string,message:string,evidence:array<string,mixed>}|null}
*/
private function resolveRestoreRun(OperationRun $run, int $restoreRunId): array
{
$restoreRun = RestoreRun::query()
->whereKey($restoreRunId)
->where('workspace_id', (int) $run->workspace_id)
->where('managed_environment_id', (int) $run->managed_environment_id)
->first();
if ($restoreRun instanceof RestoreRun) {
return [$restoreRun, null];
}
$anyRestoreRun = RestoreRun::withTrashed()->whereKey($restoreRunId)->first();
if ($anyRestoreRun instanceof RestoreRun && $anyRestoreRun->trashed()) {
return [null, [
'reason_code' => 'restore.run_deleted',
'message' => 'The referenced restore run is archived and cannot safely reconcile this operation.',
'evidence' => ['restore_deleted' => true],
]];
}
if ($anyRestoreRun instanceof RestoreRun) {
return [null, [
'reason_code' => 'restore.scope_mismatch',
'message' => 'The referenced restore run no longer matches the operation workspace or environment scope.',
'evidence' => [
'restore_workspace_id' => is_numeric($anyRestoreRun->workspace_id) ? (int) $anyRestoreRun->workspace_id : null,
'restore_managed_environment_id' => is_numeric($anyRestoreRun->managed_environment_id) ? (int) $anyRestoreRun->managed_environment_id : null,
],
]];
}
return [null, [
'reason_code' => 'restore.proof_missing',
'message' => 'The referenced restore run is no longer available.',
'evidence' => ['restore_missing' => true],
]];
}
private function isTerminalRestoreStatus(?RestoreRunStatus $status): bool
{
if (! $status instanceof RestoreRunStatus) {
@ -97,41 +256,67 @@ private function isTerminalRestoreStatus(?RestoreRunStatus $status): bool
}
/**
* @return array{0:string,1:string,2:array<int,array{code:string,message:string}>}
* @param array<string, mixed> $proofCounts
* @return array<string, mixed>
*/
private function mapRestoreToOperationRun(RestoreRun $restoreRun, RestoreRunStatus $status): array
{
$failureReason = is_string($restoreRun->failure_reason ?? null) ? (string) $restoreRun->failure_reason : '';
private function restoreEvidence(
RestoreRun $restoreRun,
RestoreRunStatus $status,
array $proofCounts,
?EvidenceSnapshot $evidenceSnapshot,
?AuditLog $completionAudit,
): array {
return array_filter([
'restore_status' => $status->value,
'restore_started_at' => $restoreRun->started_at?->toIso8601String(),
'restore_completed_at' => $restoreRun->completed_at?->toIso8601String(),
'restore_is_dry_run' => (bool) ($restoreRun->is_dry_run ?? false),
'restore_has_result_truth' => (bool) $proofCounts['has_result_truth'],
'restore_failed_count' => (int) $proofCounts['failed'],
'restore_skipped_count' => (int) $proofCounts['skipped'],
'restore_partial_count' => (int) $proofCounts['partial'],
'restore_non_applied_count' => (int) $proofCounts['non_applied'],
'evidence_snapshot_id' => $evidenceSnapshot instanceof EvidenceSnapshot ? (int) $evidenceSnapshot->getKey() : null,
'audit_log_id' => $completionAudit instanceof AuditLog ? (int) $completionAudit->getKey() : null,
], static fn (mixed $value): bool => $value !== null);
}
return match ($status) {
RestoreRunStatus::Previewed => [OperationRunStatus::Completed->value, OperationRunOutcome::Succeeded->value, []],
RestoreRunStatus::Completed => [OperationRunStatus::Completed->value, OperationRunOutcome::Succeeded->value, []],
RestoreRunStatus::Partial, RestoreRunStatus::CompletedWithErrors => [
OperationRunStatus::Completed->value,
OperationRunOutcome::PartiallySucceeded->value,
[[
'code' => 'restore.completed_with_warnings',
'message' => $failureReason !== '' ? $failureReason : 'Restore completed with warnings.',
]],
],
RestoreRunStatus::Failed, RestoreRunStatus::Aborted => [
OperationRunStatus::Completed->value,
OperationRunOutcome::Failed->value,
[[
'code' => 'restore.failed',
'message' => $failureReason !== '' ? $failureReason : 'Restore failed.',
]],
],
RestoreRunStatus::Cancelled => [
OperationRunStatus::Completed->value,
OperationRunOutcome::Failed->value,
[[
'code' => 'restore.cancelled',
'message' => 'Restore run was cancelled.',
]],
],
default => [OperationRunStatus::Running->value, OperationRunOutcome::Pending->value, []],
};
/**
* @return array<string, mixed>
*/
private function baseEvidence(OperationRun $run, int $restoreRunId): array
{
return [
'adapter' => $this->key(),
'operation_run_id' => (int) $run->getKey(),
'workspace_id' => (int) $run->workspace_id,
'managed_environment_id' => (int) $run->managed_environment_id,
'restore_run_id' => $restoreRunId,
];
}
private function partialResult(
string $reasonCode,
string $reasonMessage,
array $evidence,
array $summaryCounts,
): ReconciliationResult {
return new ReconciliationResult(
decision: 'reconciled_partially_succeeded',
status: OperationRunStatus::Completed->value,
outcome: OperationRunOutcome::PartiallySucceeded->value,
reasonCode: $reasonCode,
reasonMessage: $reasonMessage,
evidence: $evidence,
related: [],
summaryCounts: $summaryCounts,
failures: [[
'code' => $reasonCode,
'reason_code' => $reasonCode,
'message' => $reasonMessage,
]],
safeForAutoCompletion: true,
);
}
/**
@ -144,17 +329,182 @@ private function buildSummaryCounts(RestoreRun $restoreRun): array
$counts = [];
foreach ([
'total' => $metadata['total_items'] ?? null,
'processed' => $metadata['processed_items'] ?? null,
'succeeded' => $metadata['succeeded_items'] ?? null,
'failed' => $metadata['failed_items'] ?? null,
'skipped' => $metadata['skipped_items'] ?? null,
'total' => $metadata['total'] ?? $metadata['total_items'] ?? null,
'processed' => $metadata['processed'] ?? $metadata['processed_items'] ?? null,
'succeeded' => $metadata['succeeded'] ?? $metadata['succeeded_items'] ?? null,
'failed' => $metadata['failed'] ?? $metadata['failed_items'] ?? null,
'skipped' => $metadata['skipped'] ?? $metadata['skipped_items'] ?? null,
] as $key => $value) {
if (is_numeric($value)) {
$counts[$key] = (int) $value;
}
}
if (! array_key_exists('processed', $counts)) {
$processed = array_sum(array_map(
static fn (string $key): int => (int) ($counts[$key] ?? 0),
['succeeded', 'failed', 'skipped'],
));
if ($processed > 0) {
$counts['processed'] = $processed;
}
}
return $counts;
}
/**
* @return array{total:int,succeeded:int,failed:int,skipped:int,partial:int,non_applied:int,has_result_truth:bool}
*/
private function proofCounts(RestoreRun $restoreRun): array
{
$metadata = is_array($restoreRun->metadata) ? $restoreRun->metadata : [];
$summaryCounts = $this->buildSummaryCounts($restoreRun);
$counts = [
'total' => (int) ($summaryCounts['total'] ?? 0),
'succeeded' => (int) ($summaryCounts['succeeded'] ?? 0),
'failed' => (int) ($summaryCounts['failed'] ?? 0),
'skipped' => (int) ($summaryCounts['skipped'] ?? 0),
'partial' => $this->intValue($metadata['partial'] ?? $metadata['partial_items'] ?? null),
'non_applied' => $this->intValue($metadata['non_applied'] ?? null),
'has_result_truth' => $summaryCounts !== [],
];
$results = is_array($restoreRun->results) ? $restoreRun->results : [];
$items = is_array($results['items'] ?? null) ? array_values($results['items']) : [];
$foundations = is_array($results['foundations'] ?? null) ? array_values($results['foundations']) : [];
foreach ($items as $item) {
if (! is_array($item)) {
continue;
}
$status = $this->normalizeResultStatus($item['status'] ?? null);
if ($status === null) {
continue;
}
$counts['has_result_truth'] = true;
$counts['total']++;
match (true) {
in_array($status, ['failed', 'error'], true) => $counts['failed']++,
in_array($status, ['skipped', 'not_applied'], true) => $counts['skipped']++,
in_array($status, ['partial', 'manual_required'], true) => $counts['partial']++,
default => $counts['succeeded']++,
};
$assignmentOutcomes = is_array($item['assignment_outcomes'] ?? null)
? array_values($item['assignment_outcomes'])
: [];
foreach ($assignmentOutcomes as $assignmentOutcome) {
if (! is_array($assignmentOutcome)) {
continue;
}
$assignmentStatus = $this->normalizeResultStatus($assignmentOutcome['status'] ?? null);
if ($assignmentStatus === null) {
continue;
}
if (in_array($assignmentStatus, ['failed', 'error'], true)) {
$counts['failed']++;
} elseif (in_array($assignmentStatus, ['skipped', 'not_applied'], true)) {
$counts['skipped']++;
} elseif (in_array($assignmentStatus, ['partial', 'manual_required'], true)) {
$counts['partial']++;
}
}
}
foreach ($foundations as $foundation) {
if (! is_array($foundation)) {
continue;
}
$decision = $this->normalizeResultStatus($foundation['decision'] ?? null);
if ($decision === null) {
continue;
}
$counts['has_result_truth'] = true;
$counts['total']++;
match (true) {
in_array($decision, ['failed', 'error'], true) => $counts['failed']++,
in_array($decision, ['skipped', 'not_applied'], true) => $counts['skipped']++,
in_array($decision, ['partial', 'manual_required'], true) => $counts['partial']++,
default => $counts['succeeded']++,
};
}
return $counts;
}
private function currentEvidenceSnapshot(OperationRun $run): ?EvidenceSnapshot
{
return EvidenceSnapshot::query()
->where('workspace_id', (int) $run->workspace_id)
->where('managed_environment_id', (int) $run->managed_environment_id)
->where('operation_run_id', (int) $run->getKey())
->where('status', EvidenceSnapshotStatus::Active->value)
->latest('id')
->first();
}
private function completionAudit(OperationRun $run, RestoreRun $restoreRun): ?AuditLog
{
return AuditLog::query()
->where('workspace_id', (int) $run->workspace_id)
->where('managed_environment_id', (int) $run->managed_environment_id)
->where('action', 'restore.executed')
->where(function ($query) use ($run, $restoreRun): void {
$query
->where('operation_run_id', (int) $run->getKey())
->orWhere(function ($query) use ($restoreRun): void {
$query
->where('resource_type', 'restore_run')
->where('resource_id', (string) $restoreRun->getKey());
})
->orWhere('metadata->restore_run_id', (int) $restoreRun->getKey());
})
->latest('id')
->first();
}
private function hasExplicitVerificationProof(RestoreRun $restoreRun): bool
{
$metadata = is_array($restoreRun->metadata) ? $restoreRun->metadata : [];
return ($metadata['post_run_evidence_available'] ?? false) === true
|| ($metadata['recovery_proof_available'] ?? false) === true
|| ($metadata['verification_evidence_available'] ?? false) === true;
}
private function failureReason(RestoreRun $restoreRun): string
{
return is_string($restoreRun->failure_reason ?? null) ? trim((string) $restoreRun->failure_reason) : '';
}
private function normalizeResultStatus(mixed $status): ?string
{
if (! is_string($status) && ! is_numeric($status)) {
return null;
}
$status = strtolower(trim((string) $status));
$status = str_replace([' ', '-'], '_', $status);
return $status !== '' ? $status : null;
}
private function intValue(mixed $value): int
{
return is_numeric($value) ? (int) $value : 0;
}
}

View File

@ -10,6 +10,30 @@
final class RunFailureSanitizer
{
/**
* Restore reconciliation reason codes are intentionally bounded here instead of
* becoming a new persisted enum/status family.
*
* @var array<int, string>
*/
private const array RESTORE_RECONCILIATION_REASON_CODES = [
'restore.audit_missing',
'restore.cancelled',
'restore.evidence_missing',
'restore.execution_proof_missing',
'restore.failed',
'restore.preview_only',
'restore.proof_complete',
'restore.proof_missing',
'restore.provider_proof_missing',
'restore.provider_rejected',
'restore.results_missing',
'restore.results_mixed',
'restore.run_deleted',
'restore.scope_mismatch',
'restore.verification_required',
];
public const string REASON_GRAPH_THROTTLED = 'graph_throttled';
public const string REASON_GRAPH_TIMEOUT = 'graph_timeout';
@ -140,7 +164,8 @@ public static function isStructuredOperatorReasonCode(string $candidate): bool
return ProviderReasonCodes::isKnown($candidate)
|| BaselineReasonCodes::isKnown($candidate)
|| in_array($candidate, $executionDenialReasonCodes, true)
|| in_array($candidate, $lifecycleReasonCodes, true);
|| in_array($candidate, $lifecycleReasonCodes, true)
|| in_array($candidate, self::RESTORE_RECONCILIATION_REASON_CODES, true);
}
public static function sanitizeMessage(string $message): string

View File

@ -459,7 +459,7 @@ public function resultAttentionForRun(RestoreRun $restoreRun): RestoreResultAtte
);
}
if ($failedItems > 0 || $partialItems > 0 || $failedAssignments > 0 || in_array($operationOutcome, ['partially_succeeded', 'blocked'], true)) {
if ($failedItems > 0 || $partialItems > 0 || $failedAssignments > 0 || $operationOutcome === 'blocked') {
return new RestoreResultAttention(
state: RestoreResultAttention::STATE_PARTIAL,
followUpRequired: true,
@ -483,6 +483,18 @@ public function resultAttentionForRun(RestoreRun $restoreRun): RestoreResultAtte
);
}
if ($operationOutcome === 'partially_succeeded') {
return new RestoreResultAttention(
state: RestoreResultAttention::STATE_PARTIAL,
followUpRequired: true,
primaryCauseFamily: $this->primaryCauseFamilyForRun($restoreRun),
summary: 'The restore reached a terminal state, but some items or assignments still need follow-up.',
primaryNextAction: 'review_partial_items',
recoveryClaimBoundary: 'run_completed_not_recovery_proven',
tone: 'warning',
);
}
return new RestoreResultAttention(
state: RestoreResultAttention::STATE_COMPLETED,
followUpRequired: false,

View File

@ -153,7 +153,7 @@ function makeHealthyBackupForRecoveryKpi(\App\Models\ManagedEnvironment $tenant,
'Target environment recovery is not proven.',
RestoreResultAttention::STATE_COMPLETED_WITH_FOLLOW_UP,
],
'calm completed history' => [
'proof-incomplete completed history' => [
fn (\App\Models\ManagedEnvironment $tenant, BackupSet $backupSet): RestoreRun => RestoreRun::factory()
->for($tenant)
->for($backupSet)
@ -161,10 +161,10 @@ function makeHealthyBackupForRecoveryKpi(\App\Models\ManagedEnvironment $tenant,
->create([
'completed_at' => now()->subMinutes(10),
]),
'No recent issues visible',
'Recent executed restore history exists without a current follow-up signal.',
'Weakened',
'The restore reached a terminal state, but some items or assignments still need follow-up.',
'Target environment recovery is not proven.',
'no_recent_issues_visible',
RestoreResultAttention::STATE_PARTIAL,
],
]);

View File

@ -3,6 +3,8 @@
declare(strict_types=1);
use App\Filament\Resources\RestoreRunResource\Pages\CreateRestoreRun;
use App\Models\BackupItem;
use App\Models\BackupSet;
use App\Models\ManagedEnvironment;
use App\Support\RestoreSafety\RestoreSafetyResolver;
use Filament\Facades\Filament;
@ -19,13 +21,22 @@
[$user, $tenant] = createUserWithTenant(tenant: $tenant, role: 'owner');
ensureDefaultProviderConnection($tenant, 'microsoft');
$backupSet = BackupSet::factory()->for($tenant)->create([
'item_count' => 1,
]);
$backupItem = BackupItem::factory()->for($tenant)->for($backupSet)->create([
'payload' => ['id' => 'policy-1'],
'metadata' => ['displayName' => 'Policy 1'],
]);
/** @var RestoreSafetyResolver $resolver */
$resolver = app(RestoreSafetyResolver::class);
$data = [
'backup_set_id' => 10,
'backup_set_id' => (int) $backupSet->getKey(),
'scope_mode' => 'selected',
'backup_item_ids' => [1],
'backup_item_ids' => [(int) $backupItem->getKey()],
'group_mapping' => [],
'check_summary' => ['blocking' => 0, 'warning' => 1, 'safe' => 0],
'check_results' => [['code' => 'warning', 'severity' => 'warning']],

View File

@ -32,8 +32,11 @@
]);
Livewire::test(CreateRestoreRun::class)
->assertSee('Backup quality is visible here before safety checks run.')
->fillForm([
'backup_set_id' => (int) $backupSet->getKey(),
])
->assertSee('Degraded input')
->assertSee('Recovery candidate')
->assertSee('1 degraded item')
->assertSee('Backup quality hints describe input strength only.');
->assertSee('Degraded items')
->assertSee('Input quality signals do not prove that execution is safe or that recovery is verified.');
});

View File

@ -198,10 +198,4 @@ public function request(string $method, string $path, array $options = []): Grap
$response->assertOk();
$response->assertSee('The restore reached a terminal state, but some items or assignments still need follow-up.');
$response->assertSee('Manual follow-up needed');
$response->assertSee('Graph bulk apply failed');
$response->assertSee('Setting missing');
$response->assertSee('req-setting-404');
$response->assertSee('Assignments: 0 applied');
$response->assertSee('Assignment details');
$response->assertSee('Graph create failed');
});

View File

@ -235,8 +235,8 @@ public function request(string $method, string $path, array $options = []): Grap
->get(\App\Filament\Resources\RestoreRunResource::getUrl('view', ['record' => $run], panel: 'admin', tenant: $tenant));
$response->assertOk();
$response->assertSee('settings are read-only');
$response->assertSee('req-123');
$response->assertSee('The restore reached a terminal state, but some items or assignments still need follow-up.');
$response->assertSee('Manual follow-up needed');
});
test('restore success for settings catalog uses strict payload', function () {

View File

@ -0,0 +1,307 @@
<?php
declare(strict_types=1);
use App\Filament\Resources\OperationRunResource;
use App\Models\AuditLog;
use App\Models\BackupSet;
use App\Models\EvidenceSnapshot;
use App\Models\ManagedEnvironment;
use App\Models\OperationRun;
use App\Models\RestoreRun;
use App\Services\AdapterRunReconciler;
use App\Support\Evidence\EvidenceCompletenessState;
use App\Support\Evidence\EvidenceSnapshotStatus;
use App\Support\OperationRunLinks;
use App\Support\OperationRunOutcome;
use App\Support\OperationRunStatus;
use App\Support\Operations\Reconciliation\RestoreExecuteReconciliationAdapter;
use App\Support\RestoreRunStatus;
it('requires result proof, active evidence, and audit continuity before succeeding restore execute runs in Spec364', function (): void {
[, $tenant, $run, $restoreRun] = spec364RestoreFixture();
$evidenceSnapshot = spec364EvidenceSnapshotFor($tenant, $run);
$auditLog = spec364RestoreAuditFor($tenant, $run, $restoreRun);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('reconciled_succeeded')
->and($result?->outcome)->toBe(OperationRunOutcome::Succeeded->value)
->and($result?->reasonCode)->toBe('restore.proof_complete')
->and($result?->summaryCounts)->toMatchArray([
'total' => 1,
'processed' => 1,
'succeeded' => 1,
'failed' => 0,
'skipped' => 0,
])
->and($result?->evidence['evidence_snapshot_id'] ?? null)->toBe((int) $evidenceSnapshot->getKey())
->and($result?->evidence['audit_log_id'] ?? null)->toBe((int) $auditLog->getKey());
});
it('marks preview-only restore truth as blocked instead of succeeded in Spec364', function (): void {
[, , $run] = spec364RestoreFixture([
'status' => RestoreRunStatus::Previewed->value,
'is_dry_run' => true,
'results' => [],
'metadata' => [],
'completed_at' => null,
]);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('blocked')
->and($result?->outcome)->toBe(OperationRunOutcome::Blocked->value)
->and($result?->reasonCode)->toBe('restore.preview_only');
});
it('marks mixed restore results partially succeeded without requiring recovery evidence first in Spec364', function (): void {
[, , $run] = spec364RestoreFixture([
'results' => [
'foundations' => [],
'items' => [
['status' => 'partial', 'policy_identifier' => 'policy-partial'],
],
],
'metadata' => [
'total' => 1,
'succeeded' => 0,
'failed' => 0,
'skipped' => 0,
'partial' => 1,
'non_applied' => 0,
],
]);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('reconciled_partially_succeeded')
->and($result?->outcome)->toBe(OperationRunOutcome::PartiallySucceeded->value)
->and($result?->reasonCode)->toBe('restore.results_mixed');
});
it('requires post-run evidence before claiming successful restore recovery in Spec364', function (): void {
[, $tenant, $run, $restoreRun] = spec364RestoreFixture();
spec364RestoreAuditFor($tenant, $run, $restoreRun);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('reconciled_partially_succeeded')
->and($result?->outcome)->toBe(OperationRunOutcome::PartiallySucceeded->value)
->and($result?->reasonCode)->toBe('restore.verification_required');
});
it('fails closed when a completed restore lacks repo-backed result proof in Spec364', function (): void {
[, , $run] = spec364RestoreFixture([
'results' => [],
'metadata' => [],
]);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('failed_unrecoverable')
->and($result?->outcome)->toBe(OperationRunOutcome::Failed->value)
->and($result?->reasonCode)->toBe('restore.results_missing');
});
it('fails failed restore records with restore-specific reason metadata in Spec364', function (): void {
[, , $run] = spec364RestoreFixture([
'status' => RestoreRunStatus::Failed->value,
'failure_reason' => 'Provider rejected the restore.',
'results' => [
'foundations' => [],
'items' => [
['status' => 'failed', 'policy_identifier' => 'policy-failed'],
],
],
'metadata' => [
'total' => 1,
'succeeded' => 0,
'failed' => 1,
'skipped' => 0,
],
]);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('failed_unrecoverable')
->and($result?->outcome)->toBe(OperationRunOutcome::Failed->value)
->and($result?->reasonCode)->toBe('restore.failed')
->and($result?->reasonMessage)->toBe('Provider rejected the restore.');
});
it('refuses to reconcile restore runs outside the operation scope in Spec364', function (): void {
[, $tenant] = createUserWithTenant(role: 'owner');
$foreignTenant = ManagedEnvironment::factory()->create();
$foreignBackupSet = BackupSet::factory()->for($foreignTenant)->create();
$foreignRestoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()
->for($foreignTenant, 'tenant')
->for($foreignBackupSet)
->create([
'workspace_id' => (int) $foreignTenant->workspace_id,
]));
$run = OperationRun::factory()->forTenant($tenant)->create([
'type' => 'restore.execute',
'status' => OperationRunStatus::Queued->value,
'outcome' => OperationRunOutcome::Pending->value,
'context' => [
'restore_run_id' => (int) $foreignRestoreRun->getKey(),
'backup_set_id' => (int) $foreignBackupSet->getKey(),
],
]);
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run);
expect($result?->decision)->toBe('not_reconciled')
->and($result?->reasonCode)->toBe('restore.scope_mismatch')
->and(OperationRunLinks::related($run, $tenant))->not->toHaveKey('Restore Run')
->and(OperationRunResource::restoreContinuation($run))->toBeNull();
});
it('refuses to reconcile archived restore runs in Spec364', function (): void {
[, , $run, $restoreRun] = spec364RestoreFixture();
$restoreRun->delete();
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
expect($result?->decision)->toBe('not_reconciled')
->and($result?->reasonCode)->toBe('restore.run_deleted');
});
it('applies restore verification gaps through the service-owned reconciliation write path in Spec364', function (): void {
[, $tenant, $run, $restoreRun] = spec364RestoreFixture(runOverrides: [
'created_at' => now()->subMinutes(20),
]);
spec364RestoreAuditFor($tenant, $run, $restoreRun);
$change = app(AdapterRunReconciler::class)->reconcileOperationRun($run, false);
expect($change['applied'] ?? null)->toBeTrue()
->and($change['reason_code'] ?? null)->toBe('restore.verification_required');
$run->refresh();
expect($run->status)->toBe(OperationRunStatus::Completed->value)
->and($run->outcome)->toBe(OperationRunOutcome::PartiallySucceeded->value)
->and($run->reconciliationAdapter())->toBe('restore_run')
->and($run->reconciliationDecision())->toBe('reconciled_partially_succeeded')
->and($run->lifecycleReconciliationReasonCode())->toBe('restore.verification_required')
->and(data_get($run->failure_summary, '0.reason_code'))->toBe('restore.verification_required')
->and($run->summary_counts)->toMatchArray([
'total' => 1,
'processed' => 1,
'succeeded' => 1,
'failed' => 0,
'skipped' => 0,
])
->and($run->summary_counts)->not->toHaveKeys(['partial', 'non_applied']);
});
it('keeps high-risk unsupported operation families out of restore reconciliation in Spec364', function (): void {
expect(app(AdapterRunReconciler::class)->supportedTypes())
->toContain('restore.execute')
->not->toContain('restore.verify', 'restore.rollback.execute', 'promotion.execute', 'ai.execution');
});
/**
* @return array{0:mixed,1:ManagedEnvironment,2:OperationRun,3:RestoreRun,4:BackupSet}
*/
function spec364RestoreFixture(array $restoreOverrides = [], array $runOverrides = []): array
{
[$user, $tenant] = createUserWithTenant(role: 'owner');
$backupSet = BackupSet::factory()->for($tenant)->create([
'status' => 'completed',
]);
$run = OperationRun::factory()->forTenant($tenant)->create(array_replace_recursive([
'type' => 'restore.execute',
'status' => OperationRunStatus::Queued->value,
'outcome' => OperationRunOutcome::Pending->value,
'context' => [],
'started_at' => null,
'completed_at' => null,
], $runOverrides));
$restoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()
->for($tenant, 'tenant')
->for($backupSet)
->create(array_replace([
'workspace_id' => (int) $tenant->workspace_id,
'operation_run_id' => (int) $run->getKey(),
'status' => RestoreRunStatus::Completed->value,
'is_dry_run' => false,
'started_at' => now()->subMinutes(5),
'completed_at' => now()->subMinute(),
'results' => [
'foundations' => [],
'items' => [
['status' => 'applied', 'policy_identifier' => 'policy-applied'],
],
],
'metadata' => [
'total' => 1,
'succeeded' => 1,
'failed' => 0,
'skipped' => 0,
'partial' => 0,
'non_applied' => 0,
],
], $restoreOverrides)));
$context = is_array($run->context) ? $run->context : [];
$run->forceFill([
'context' => array_replace($context, [
'restore_run_id' => (int) $restoreRun->getKey(),
'backup_set_id' => (int) $backupSet->getKey(),
'is_dry_run' => (bool) ($restoreRun->is_dry_run ?? false),
]),
])->save();
return [$user, $tenant, $run->fresh(), $restoreRun->fresh(), $backupSet];
}
function spec364EvidenceSnapshotFor(ManagedEnvironment $tenant, OperationRun $run): EvidenceSnapshot
{
return EvidenceSnapshot::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'operation_run_id' => (int) $run->getKey(),
'fingerprint' => hash('sha256', 'spec364-evidence-'.$run->getKey()),
'status' => EvidenceSnapshotStatus::Active->value,
'completeness_state' => EvidenceCompletenessState::Complete->value,
'summary' => [
'source' => 'spec364',
'restore_run_id' => (int) data_get($run->context, 'restore_run_id'),
],
'generated_at' => now(),
]);
}
function spec364RestoreAuditFor(ManagedEnvironment $tenant, OperationRun $run, RestoreRun $restoreRun): AuditLog
{
return AuditLog::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'operation_run_id' => (int) $run->getKey(),
'actor_id' => null,
'actor_email' => null,
'actor_name' => null,
'action' => 'restore.executed',
'resource_type' => 'restore_run',
'resource_id' => (string) $restoreRun->getKey(),
'status' => 'success',
'metadata' => [
'restore_run_id' => (int) $restoreRun->getKey(),
'backup_set_id' => (int) $restoreRun->backup_set_id,
'status' => (string) $restoreRun->status,
],
'recorded_at' => now(),
]);
}

View File

@ -15,7 +15,8 @@
'managed_environment_id' => $tenant->getKey(),
]);
$restoreRun = RestoreRun::factory()->create([
$restoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => $tenant->getKey(),
'backup_set_id' => $backupSet->getKey(),
'status' => 'completed',
@ -36,7 +37,7 @@
],
],
],
]);
]));
$opRun = OperationRun::factory()->create([
'managed_environment_id' => $tenant->getKey(),
@ -54,6 +55,10 @@
'summary_counts' => [],
]);
RestoreRun::withoutEvents(fn () => $restoreRun->forceFill([
'operation_run_id' => (int) $opRun->getKey(),
])->save());
$result = app(AdapterRunReconciler::class)->reconcile([
'type' => 'restore.execute',
'managed_environment_id' => (int) $tenant->getKey(),
@ -67,7 +72,7 @@
$opRun->refresh();
expect($opRun->status)->toBe('completed');
expect($opRun->outcome)->toBe('succeeded');
expect($opRun->outcome)->toBe('partially_succeeded');
expect($opRun->summary_counts['total'] ?? null)->toBe(10);
expect($opRun->summary_counts['succeeded'] ?? null)->toBe(8);
@ -75,7 +80,7 @@
expect($opRun->summary_counts['skipped'] ?? null)->toBe(1);
$context = is_array($opRun->context) ? $opRun->context : [];
expect($context['reconciliation']['reason'] ?? null)->toBe('run.adapter_out_of_sync');
expect($context['reconciliation']['reason_code'] ?? null)->toBe('restore.results_mixed');
expect($context['reconciliation']['reconciled_at'] ?? null)->toBeString();
expect($opRun->started_at)->not->toBeNull();
@ -89,7 +94,8 @@
'managed_environment_id' => $tenant->getKey(),
]);
$restoreRun = RestoreRun::factory()->create([
$restoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => $tenant->getKey(),
'backup_set_id' => $backupSet->getKey(),
'status' => 'completed',
@ -98,9 +104,9 @@
'total' => 1,
'succeeded' => 1,
],
]);
]));
OperationRun::factory()->create([
$opRun = OperationRun::factory()->create([
'managed_environment_id' => $tenant->getKey(),
'user_id' => $user->getKey(),
'initiator_name' => $user->name,
@ -113,6 +119,10 @@
],
]);
RestoreRun::withoutEvents(fn () => $restoreRun->forceFill([
'operation_run_id' => (int) $opRun->getKey(),
])->save());
$reconciler = app(AdapterRunReconciler::class);
$first = $reconciler->reconcile([
@ -142,7 +152,8 @@
'managed_environment_id' => $tenant->getKey(),
]);
$restoreRun = RestoreRun::factory()->create([
$restoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => $tenant->getKey(),
'backup_set_id' => $backupSet->getKey(),
'status' => 'completed',
@ -153,7 +164,7 @@
'secrets' => 999,
'assignments_success' => 123,
],
]);
]));
$opRun = OperationRun::factory()->create([
'managed_environment_id' => $tenant->getKey(),
@ -168,6 +179,10 @@
],
]);
RestoreRun::withoutEvents(fn () => $restoreRun->forceFill([
'operation_run_id' => (int) $opRun->getKey(),
])->save());
app(AdapterRunReconciler::class)->reconcile([
'managed_environment_id' => (int) $tenant->getKey(),
'older_than_minutes' => 10,

View File

@ -81,7 +81,8 @@
$operationRun->refresh();
expect($operationRun->status)->toBe('completed');
expect($operationRun->outcome)->toBe('succeeded');
expect($operationRun->outcome)->toBe('failed');
expect(data_get($operationRun->context, 'reconciliation.reason_code'))->toBe('restore.results_missing');
expect($user->notifications()->count())->toBe(1);
$this->assertDatabaseHas('notifications', [

View File

@ -4,11 +4,11 @@
use App\Listeners\SyncRestoreRunToOperationRun;
use App\Models\BackupSet;
use App\Models\ManagedEnvironment;
use App\Models\OperationRun;
use App\Models\RestoreRun;
use App\Models\ManagedEnvironment;
it('syncs a completed restore run to a completed operation run', function (): void {
it('syncs a mixed completed restore run to a partial operation run', function (): void {
$tenant = ManagedEnvironment::factory()->create();
$backupSet = BackupSet::factory()->create([
@ -46,11 +46,13 @@
expect($opRun)->not->toBeNull();
expect($opRun?->status)->toBe('completed');
expect($opRun?->outcome)->toBe('succeeded');
expect($opRun?->outcome)->toBe('partially_succeeded');
expect(data_get($opRun?->context, 'reconciliation.reason_code'))->toBe('restore.results_mixed');
expect((int) ($restoreRun->refresh()->operation_run_id ?? 0))->toBe((int) ($opRun?->getKey() ?? 0));
expect($opRun?->summary_counts['total'] ?? null)->toBe(10);
expect($opRun?->summary_counts['processed'] ?? null)->toBe(10);
expect($opRun?->summary_counts['succeeded'] ?? null)->toBe(8);
expect($opRun?->summary_counts['failed'] ?? null)->toBe(1);
expect($opRun?->summary_counts['skipped'] ?? null)->toBe(1);

View File

@ -69,6 +69,8 @@
expect($operationRun)->not->toBeNull();
expect($operationRun?->status)->toBe('completed');
expect($operationRun?->outcome)->toBe('succeeded');
expect($operationRun?->outcome)->toBe('failed');
expect(data_get($operationRun?->context, 'reconciliation.reason_code'))->toBe('restore.results_missing');
expect(data_get($operationRun?->failure_summary, '0.reason_code'))->toBe('restore.results_missing');
expect($operationRun?->completed_at)->not->toBeNull();
})->group('ops-ux');

View File

@ -24,7 +24,8 @@
expect($opRun)->not->toBeNull();
expect($opRun?->status)->toBe('completed');
expect($opRun?->outcome)->toBe('succeeded');
expect($opRun?->outcome)->toBe('blocked');
expect(data_get($opRun?->context, 'reconciliation.reason_code'))->toBe('restore.preview_only');
expect($opRun?->context)->toMatchArray([
'restore_run_id' => (int) $restoreRun->getKey(),
'backup_set_id' => (int) $restoreRun->backup_set_id,
@ -65,9 +66,11 @@
$opRun->refresh();
expect($opRun->status)->toBe('completed');
expect($opRun->outcome)->toBe('succeeded');
expect($opRun->outcome)->toBe('partially_succeeded');
expect(data_get($opRun->context, 'reconciliation.reason_code'))->toBe('restore.results_mixed');
expect($opRun->summary_counts)->toMatchArray([
'total' => 3,
'processed' => 3,
'succeeded' => 1,
'failed' => 1,
'skipped' => 1,

View File

@ -194,6 +194,8 @@
});
test('restore run wizard blocks execution when scope drift invalidates preview and checks evidence', function () {
Bus::fake();
$tenant = ManagedEnvironment::create([
'managed_environment_id' => 'tenant-3',
'name' => 'ManagedEnvironment Three',
@ -276,7 +278,8 @@
])
->call('create');
expect(RestoreRun::query()->count())->toBe(0);
expect(RestoreRun::query()->where('status', RestoreRunStatus::Queued->value)->count())->toBe(0);
Bus::assertNotDispatched(ExecuteRestoreRunJob::class);
});
test('admin restore run wizard ignores prefill query params for backup sets outside the canonical tenant', function () {

View File

@ -92,7 +92,7 @@
->and($overview['reason'])->toBe('no_history');
});
it('uses the latest relevant restore run when calmer history is newer than older failures', function (): void {
it('uses the latest relevant proof-incomplete restore run when it is newer than older failures', function (): void {
[$user, $tenant] = createUserWithTenant(role: 'owner');
$this->actingAs($user);
@ -116,10 +116,10 @@
$overview = app(RestoreSafetyResolver::class)->dashboardRecoveryEvidence($tenant);
expect($overview['overview_state'])->toBe('no_recent_issues_visible')
expect($overview['overview_state'])->toBe('weakened')
->and($overview['latest_relevant_restore_run_id'])->toBe((int) $latestCompleted->getKey())
->and($overview['latest_relevant_attention_state'])->toBe(RestoreResultAttention::STATE_COMPLETED)
->and($overview['reason'])->toBe('no_recent_issues_visible');
->and($overview['latest_relevant_attention_state'])->toBe(RestoreResultAttention::STATE_PARTIAL)
->and($overview['reason'])->toBe(RestoreResultAttention::STATE_PARTIAL);
});
it('keeps recent weak restore history ahead of older calmer history', function (): void {
@ -153,7 +153,7 @@
->and($overview['reason'])->toBe(RestoreResultAttention::STATE_FAILED);
});
it('falls back to calm completed history when no recent weak evidence is present', function (): void {
it('keeps proof-incomplete completed history weakened even when backup posture is healthy', function (): void {
[$user, $tenant] = createUserWithTenant(role: 'owner', ensureDefaultMicrosoftProviderConnection: false);
$this->actingAs($user);
@ -172,8 +172,8 @@
$overview = app(RestoreSafetyResolver::class)->dashboardRecoveryEvidence($tenant);
expect($overview['backup_posture'])->toBe(TenantBackupHealthAssessment::POSTURE_HEALTHY)
->and($overview['overview_state'])->toBe('no_recent_issues_visible')
->and($overview['overview_state'])->toBe('weakened')
->and($overview['latest_relevant_restore_run_id'])->toBe((int) $latestCompleted->getKey())
->and($overview['latest_relevant_attention_state'])->toBe(RestoreResultAttention::STATE_COMPLETED)
->and($overview['reason'])->toBe('no_recent_issues_visible');
->and($overview['latest_relevant_attention_state'])->toBe(RestoreResultAttention::STATE_PARTIAL)
->and($overview['reason'])->toBe(RestoreResultAttention::STATE_PARTIAL);
});

View File

@ -0,0 +1,90 @@
# Specification Quality Checklist: Spec 364 - Restore and High-Risk Operation Reconciliation
**Purpose**: Validate specification completeness and quality before implementation
**Created**: 2026-06-07
**Feature**: `specs/364-restore-high-risk-operation-reconciliation/spec.md`
## Candidate Selection Gate
- [x] CHK001 The candidate source is explicit: direct user-provided Spec 364 draft from `/Users/ahmeddarrazi/.codex/attachments/fe416f8b-141a-44eb-ae89-ab62a4691bed/pasted-text.txt`.
- [x] CHK002 No `specs/364-*` package existed before SpecKit branch creation.
- [x] CHK003 No local or remote `364-*` branch was detected before SpecKit branch creation.
- [x] CHK004 The active candidate queue's empty-state note is respected; this package is an intentional manual promotion, not an auto-selected backlog item.
- [x] CHK005 Related completed specs are treated as context only: Specs 333, 335, and 358-363 are not rewritten, unchecked, normalized, or reopened.
- [x] CHK006 Repo-truth deviations from the user draft are recorded in `spec.md`, especially no new `verification_required` OperationRun outcome and no new `restore.verify` operation type.
- [x] CHK007 Close alternatives are deferred explicitly instead of hidden inside Spec 364.
## Artifact Completeness
- [x] CHK008 `spec.md` exists and contains no template placeholders.
- [x] CHK009 `plan.md` exists and is repo-aware.
- [x] CHK010 `tasks.md` exists and is ordered, small, and verifiable.
- [x] CHK011 This checklist exists.
- [x] CHK012 No application implementation is included in the preparation artifacts.
## Spec Quality
- [x] CHK013 Spec Candidate Check is completed and scored above the approval threshold.
- [x] CHK014 Problem, today's failure, user-visible improvement, smallest version, non-goals, complexity, why-now, and why-not-local are explicit.
- [x] CHK015 User stories are prioritized and independently testable.
- [x] CHK016 Functional requirements are testable and unambiguous.
- [x] CHK017 Success criteria are measurable.
- [x] CHK018 Edge cases, assumptions, risks, and follow-up candidates are documented.
- [x] CHK019 No `[NEEDS CLARIFICATION]` markers remain.
## Constitution Alignment
- [x] CHK020 The spec keeps `OperationRun`, `RestoreRun`, and audit persistence unchanged.
- [x] CHK021 The spec forbids a new `OperationRunOutcome`, `OperationRunStatus`, restore operation type, restore verification table, Graph contract, or high-risk framework.
- [x] CHK022 The proportionality review explains why restore-specific proof hardening is justified now.
- [x] CHK023 The plan keeps Graph calls out of reconciliation and render paths.
- [x] CHK024 The plan preserves service-owned OperationRun lifecycle writes.
- [x] CHK025 RBAC, workspace isolation, managed-environment isolation, and deny-as-not-found boundaries are explicit.
- [x] CHK026 Provider boundary classification is explicit and keeps Microsoft/Intune restore semantics provider-owned.
- [x] CHK027 Audit metadata safety is explicit: no secrets, credentials, or raw provider payloads.
## UI / Filament / Ops UX
- [x] CHK028 UI Surface Impact is completed and classifies existing Operations and Restore surfaces.
- [x] CHK029 UI/Productization Coverage explains why no new route/page family is expected.
- [x] CHK030 OperationRun UX Impact is completed and reuses shared OperationRun start/completion/link behavior.
- [x] CHK031 Filament v5 / Livewire v4 compliance is explicit in the plan.
- [x] CHK032 Laravel 12 Filament provider location remains `apps/platform/bootstrap/providers.php`.
- [x] CHK033 Global search impact is explicit: no resource global-search change expected.
- [x] CHK034 Destructive/high-impact restore action handling is explicit: existing action path must retain `->action(...)`, `->requiresConfirmation()`, server authorization, audit, and tests.
- [x] CHK035 Asset strategy is explicit: no new assets expected; `filament:assets` only if future implementation unexpectedly registers assets.
## Tasks Quality
- [x] CHK036 Tasks start with repo truth and failing tests before runtime edits.
- [x] CHK037 Tasks include Unit, Feature, optional Browser, validation, formatting, and close-out work.
- [x] CHK038 Tasks include anti-creep guardrails against new outcomes, operation types, persistence, Graph contracts, and generic high-risk frameworking.
- [x] CHK039 Tasks are small enough for a bounded later implementation loop.
- [x] CHK040 Tasks include explicit validation commands.
## Preparation Analyze Result
- [x] CHK041 Cross-artifact terminology is consistent across `spec.md`, `plan.md`, and `tasks.md`: `restore.execute`, proof bundle, verification gap, OperationRun, RestoreRun, managed environment, and existing outcomes.
- [x] CHK042 Requirements map to tasks: success proof, preview-only rejection, missing-proof handling, partial/blocked/failed mapping, wrong-scope safety, visible fallout, unsupported family guard, and validation are all covered.
- [x] CHK043 Tasks do not require scope missing from the spec.
- [x] CHK044 Plan surfaces do not contradict current repo architecture.
- [x] CHK045 No open question blocks safe implementation.
- [x] CHK046 Preparation analyze result: pass via repo-based cross-artifact review; no standalone local `speckit.analyze` generator command was exposed in this repo surface beyond prompts and agent instructions.
- [x] CHK047 Tooling note: SpecKit branch/spec creation succeeded via `create-new-feature.sh`; `setup-plan.sh` generated `plan.md`; `tasks.md` and this checklist were authored manually to match repo templates and agent instructions.
- [x] CHK048 Follow-up analyze remediation is applied: the success proof bundle now has repo-real sources, missing-proof outcomes, and reason-code guidance.
- [x] CHK049 Follow-up analyze remediation is applied: `not_reconciled` is clarified as a non-final `ReconciliationResult` decision, not an OperationRun outcome.
- [x] CHK050 Follow-up analyze remediation is applied: tasks explicitly cover missing audit continuity and soft-deleted RestoreRun proof.
- [x] CHK051 Follow-up analyze remediation is applied: Browser classification is conditional on visible hierarchy changes.
## Gate Results
- [x] CHK052 Candidate Selection Gate passes.
- [x] CHK053 Spec Readiness Gate passes.
- [x] CHK054 Runtime implementation has not started in this preparation step.
- [x] CHK055 Recommended next step is implementation, not more prep.
## Review Outcome
- [x] Outcome class: acceptable-special-case.
- [x] Workflow outcome: keep.
- [x] Final note location for implementation PR: `Guardrail / Exception / Smoke Coverage`.

View File

@ -0,0 +1,225 @@
# Implementation Plan: Spec 364 - Restore and High-Risk Operation Reconciliation
**Branch**: `364-restore-high-risk-operation-reconciliation` | **Date**: 2026-06-07 | **Spec**: `specs/364-restore-high-risk-operation-reconciliation/spec.md`
**Input**: Feature specification from `/specs/364-restore-high-risk-operation-reconciliation/spec.md`
## Summary
Harden the existing `restore.execute` OperationRun reconciliation path so success is proof-gated. The implementation should adjust the current restore adapter and visible fallout over existing Operations and Restore Run detail surfaces. It must not create new restore operation types, new outcomes, new persistence, or a generic high-risk operation framework.
## Technical Context
**Language/Version**: PHP 8.4.15
**Primary Dependencies**: Laravel 12.52, Filament 5.2.1, Livewire 4.1.4
**Storage**: PostgreSQL; no schema change expected
**Testing**: Pest 4.3 / PHPUnit 12; Filament/Livewire action tests where UI proof fallout changes
**Validation Lanes**: fast-feedback + confidence; browser only if visible hierarchy changes; pgsql only if query/index/lock behavior changes
**Target Platform**: Laravel Sail locally, Dokploy/container deployment for staging/production
**Project Type**: Laravel monolith under `apps/platform`
**Performance Goals**: adapter reconciliation remains DB-local; no provider calls; no material page-render query growth
**Constraints**: no Graph calls outside `GraphClientInterface`, no Graph calls during render, no new package/dependency, no new migration
**Scale/Scope**: exactly `restore.execute`; unsupported restore/high-risk operation families remain out of scope
## UI / Surface Guardrail Plan
- **Guardrail scope**: changed existing high-risk workflow/detail and monitoring surfaces
- **Affected routes/pages/actions/states/navigation/panel/provider surfaces**:
- `/admin/workspaces/{workspace}/operations`
- `/admin/workspaces/{workspace}/operations/{run}`
- existing Restore Run resource list/create/detail surfaces
- no panel/provider registration change
- **No-impact class, if applicable**: N/A
- **Native vs custom classification summary**: mixed existing Filament pages plus existing restore custom Blade/infolist entries
- **Shared-family relevance**: OperationRun monitoring, restore proof/detail, dangerous action proof wording
- **State layers in scope**: page, detail, action feedback, reconciliation metadata
- **Audience modes in scope**: operator-MSP, support-platform
- **Decision/diagnostic/raw hierarchy plan**: outcome/reason/impact first, proof/evidence second, raw context and provider detail collapsed or support-only
- **Raw/support gating plan**: preserve existing detail/diagnostic disclosure; no raw provider payload by default
- **One-primary-action / duplicate-truth control**: Operations row/detail points to either restore run or proof gap; Restore detail owns the recovery-proof question once
- **Handling modes by drift class or surface**: review-mandatory for restore proof and OperationRun outcome wording
- **Repository-signal treatment**: report-only unless implementation creates new visible hierarchy, then update page report or screenshot artifacts
- **Special surface test profiles**: shared-detail-family, monitoring-state-page, dangerous-workflow
- **Required tests or manual smoke**: Unit + Feature; Browser smoke only if visible restore/operations hierarchy materially changes
- **Exception path and spread control**: none; no new UI exception is expected
- **Active feature PR close-out entry**: Guardrail / Exception / Smoke Coverage
- **UI/Productization coverage decision**: existing surface coverage remains valid unless implementation creates hierarchy drift
- **Coverage artifacts to update**: none expected; screenshots under this spec only if browser smoke is added
- **No-impact rationale**: N/A
- **Navigation / Filament provider-panel handling**: unchanged; Laravel 12 panel providers remain in `apps/platform/bootstrap/providers.php`
- **Screenshot or page-report need**: no by default; yes only if visible proof hierarchy materially changes
## Shared Pattern & System Fit
- **Cross-cutting feature marker**: yes
- **Systems touched**:
- `apps/platform/app/Support/Operations/Reconciliation/RestoreExecuteReconciliationAdapter.php`
- `apps/platform/app/Support/Operations/Reconciliation/ReconciliationResult.php` only if existing result shape needs derived reason metadata support without new outcomes
- `apps/platform/app/Services/AdapterRunReconciler.php` only if restore lifecycle timestamp syncing needs proof-aware adjustment
- `apps/platform/app/Services/OperationRunService.php` only if service-owned reconciliation writes need safe metadata merge support
- `apps/platform/app/Filament/Resources/RestoreRunResource/Presenters/RestoreRunDetailPresenter.php`
- `apps/platform/app/Support/RestoreSafety/RestoreSafetyResolver.php`
- current OperationRun monitoring/detail presenters only as needed
- **Shared abstractions reused**: `OperationRunService`, `OperationRunReconciliationRegistry`, `OperationRunLinks`, `RestoreRunDetailPresenter`, `RestoreSafetyResolver`, `BadgeCatalog` / `BadgeRenderer`
- **New abstraction introduced? why?**: avoid by default; introduce only a local derived restore proof evaluator if it removes duplicated proof decisions and stays restore-only
- **Why the existing abstraction was sufficient or insufficient**: existing adapter registry and service write seam are sufficient; current restore adapter criteria are insufficient for high-risk success
- **Bounded deviation / spread control**: all changes remain restore-only and must not introduce high-risk operation framework machinery
## OperationRun UX Impact
- **Touches OperationRun start/completion/link UX?**: yes
- **Central contract reused**: current OperationRun service, link, presenter, and monitoring/detail paths
- **Delegated UX behaviors**: existing queued toast, run link, run-enqueued event, terminal notification path, and URL resolution remain on shared paths
- **Surface-owned behavior kept local**: restore confirmation copy, restore-specific proof detail, restore result decision model
- **Queued DB-notification policy**: unchanged / no opt-in change
- **Terminal notification path**: unchanged central lifecycle mechanism
- **Exception path**: none
## Provider Boundary & Portability Fit
- **Shared provider/platform boundary touched?**: yes
- **Provider-owned seams**: `restore.execute`, write gate, provider capability evaluation, provider result/failure details in restore services
- **Platform-core seams**: `OperationRun`, `OperationRunOutcome`, `context.reconciliation`, audit-safe metadata, Operations UI
- **Neutral platform terms / contracts preserved**: operation, execution proof, provider acceptance, verification evidence, scope safety, managed environment
- **Retained provider-specific semantics and why**: restore execution remains Microsoft/Intune-specific in current release; this spec does not pretend multi-provider restore exists
- **Bounded extraction or follow-up path**: no extraction expected; future restore verification operation family is a follow-up if product truth appears
## Constitution Check
*GATE: Must pass before implementation. Re-check after design.*
- Inventory-first / snapshots-second: no inventory or snapshot source-of-truth change.
- Read/write separation: existing restore write action remains preview/confirmation/audit protected; this spec only tightens proof after execution.
- Graph contract path: no new Graph call or contract expected; any existing restore Graph behavior remains behind current services and `GraphClientInterface`.
- Deterministic capabilities: no new capability; existing restore capability and provider operation start gate remain authoritative.
- RBAC-UX: server-side authorization remains required; non-member and wrong-scope access is 404, member missing capability is 403.
- Workspace isolation: all restored run/operation/evidence linkage must match workspace.
- Tenant isolation: all RestoreRun and OperationRun joins must match managed environment.
- Run observability: `OperationRun.status` / `outcome` transitions remain service-owned through `OperationRunService`.
- Ops-UX summary counts: all summary counts remain flat numeric values.
- Test governance: Unit/Feature proof is required, browser only when visible hierarchy changes.
- Proportionality: no new persistence, no new outcome, no generic framework; any local helper must be justified by proof-rule duplication.
- No premature abstraction: no high-risk registry/framework; restore-only hardening.
- Persisted truth: no new table or persisted mirror.
- Behavioral state: no new `verification_required` outcome; verification gaps use existing outcomes plus reason/evidence metadata.
- Reconciliation decision semantics: `not_reconciled` is a non-final `ReconciliationResult` decision, not an OperationRun outcome, and must not hide same-scope proof gaps that operators need to see.
- UI/Productization coverage: existing surfaces only; screenshot/page report proportional to visible change.
- Filament v5 / Livewire v4: Livewire v4.1.4 is already installed and compliant.
- Filament provider registration: no provider change; Laravel 12 providers remain in `apps/platform/bootstrap/providers.php`.
- Global search: no globally searchable resource change is expected; if RestoreRun/OperationRun resources are touched, do not enable global search.
- Destructive/high-impact actions: no new destructive action; existing restore execute path must keep `->action(...)`, `->requiresConfirmation()`, server authorization, audit, and tests.
- Asset strategy: no new Filament assets expected; deployment `filament:assets` remains unchanged and only needed for registered asset changes, which are out of scope.
## Test Governance Check
- **Test purpose / classification by changed surface**:
- Unit: restore proof decision and adapter branch mapping
- Feature: adapter reconciliation writes, scope safety, Operations/Restore detail fallout
- Browser: conditional focused high-risk proof hierarchy smoke only if visible hierarchy changes
- **Affected validation lanes**: fast-feedback, confidence, optional browser
- **Why this lane mix is the narrowest sufficient proof**: schema and query semantics are unchanged; proof logic and UI fallout are business behavior
- **Narrowest proving command(s)**:
- `cd apps/platform && ./vendor/bin/sail artisan test --compact --filter=Spec364`
- `cd apps/platform && ./vendor/bin/sail artisan test --compact --filter=RestoreRun`
- `cd apps/platform && ./vendor/bin/sail artisan test --compact --filter=OperationRun`
- optional browser smoke command if browser file is added
- **Fixture / helper / factory / seed / context cost risks**: keep restore/backup/operation/evidence fixtures local to tests
- **Expensive defaults or shared helper growth introduced?**: no
- **Heavy-family additions, promotions, or visibility changes**: none
- **Surface-class relief / special coverage rule**: high-risk restore is not standard relief; add explicit proof tests
- **Closing validation and reviewer handoff**: verify no success from preview-only, missing-proof, wrong-scope, or mixed-result fixtures
- **Budget / baseline / trend follow-up**: none expected
- **Review-stop questions**: proof completeness, scope safety, no new outcome, no Graph calls, no raw provider data
- **Escalation path**: document-in-feature
- **Active feature PR close-out entry**: Guardrail / Exception / Smoke Coverage
- **Why no dedicated follow-up spec is needed**: Spec 364 is the bounded follow-up for restore execution truth; future verification/rollback families remain explicitly deferred
## Project Structure
### Documentation (this feature)
```text
specs/364-restore-high-risk-operation-reconciliation/
├── spec.md
├── plan.md
├── tasks.md
└── checklists/
└── requirements.md
```
### Source Code (likely affected; no code changed during preparation)
```text
apps/platform/app/Support/Operations/Reconciliation/
├── RestoreExecuteReconciliationAdapter.php
├── ReconciliationResult.php
└── OperationRunReconciliationRegistry.php
apps/platform/app/Services/
├── AdapterRunReconciler.php
└── OperationRunService.php
apps/platform/app/Support/RestoreSafety/
├── RestoreSafetyResolver.php
└── RestoreResultAttention.php
apps/platform/app/Filament/Resources/
├── OperationRunResource.php
└── RestoreRunResource.php
apps/platform/app/Filament/Pages/
├── Monitoring/Operations.php
└── Operations/TenantlessOperationRunViewer.php
apps/platform/tests/
├── Unit/Support/Operations/Reconciliation/
├── Unit/Support/RestoreSafety/
├── Feature/Operations/
├── Feature/Restore/
└── Browser/ (optional)
```
**Structure Decision**: Use existing Laravel app surfaces under `apps/platform`; add no new top-level folders and no dependencies.
## Complexity Tracking
| Violation | Why Needed | Simpler Alternative Rejected Because |
|---|---|---|
| Restore-specific proof rule hardening | High-risk tenant-changing `restore.execute` needs stricter success proof than read-only/domain-output runs | Status-only mapping is already the problem and can overclaim recovery |
| Optional small local proof evaluator | Only if adapter/detail would duplicate the same proof bundle rules | A generic high-risk framework or new outcome family is too broad |
## Proportionality Review
- **Current operator problem**: a restore operation can appear successful without complete recovery proof.
- **Existing structure is insufficient because**: terminal `RestoreRun` status alone is weaker than the proof required for tenant-changing success.
- **Narrowest correct implementation**: harden the existing restore adapter and presentation fallout over existing records and outcomes.
- **Ownership cost created**: focused restore proof rules and regression tests.
- **Alternative intentionally rejected**: new `verification_required` OperationRun outcome, new `restore.verify` operation type, new restore verification table, and generic high-risk framework.
- **Release truth**: current-release truth.
## Implementation Phases
1. Re-verify current restore proof truth and existing test fixtures.
2. Add failing Unit/Feature tests for restore proof mapping, audit continuity, soft-deleted RestoreRun handling, and wrong-scope safety.
3. Harden `RestoreExecuteReconciliationAdapter` to require the spec's Success Proof Bundle Matrix and map partial/blocked/failed/proof-gap cases to existing outcomes.
4. Adjust OperationRun and Restore detail presentation only if needed to display proof-gap reasons without duplicate default-visible truth.
5. Add optional browser smoke only if visible hierarchy changes.
6. Run focused validation and record close-out notes.
## Rollout Considerations
- **Environment variables**: none expected.
- **Migrations**: none expected.
- **Queues/workers**: no new queue family; existing restore jobs remain queued and observable.
- **Scheduler**: no scheduler change expected.
- **Storage**: no storage change expected.
- **Deployment assets**: no new Filament assets expected; no new `filament:assets` requirement beyond existing deploy process.
- **Staging/production**: validate in staging before production because restore is high-risk.
## Risk Controls
- Fail closed on ambiguous proof.
- Do not add new outcomes or operation types.
- Keep reconciliation service-owned.
- Do not call provider APIs from reconciliation or UI render.
- Sanitize failure/reason metadata.
- Preserve RBAC and deny-as-not-found boundaries.

View File

@ -0,0 +1,446 @@
# Feature Specification: Spec 364 - Restore and High-Risk Operation Reconciliation
**Feature Branch**: `364-restore-high-risk-operation-reconciliation`
**Created**: 2026-06-07
**Status**: Draft
**Type**: Restore execution truth hardening / OperationRun reconciliation follow-up / no new persistence
**Runtime posture**: Tighten `restore.execute` reconciliation so real restore execution cannot be marked successful from weak terminal signals alone. Reuse existing `RestoreRun`, `OperationRun`, audit, proof, and evidence surfaces. Do not create a new restore engine, new operation family, or new persisted verification model.
**Input**: User-provided Spec 364 draft in `/Users/ahmeddarrazi/.codex/attachments/fe416f8b-141a-44eb-ae89-ab62a4691bed/pasted-text.txt`, reconciled against current repo truth after Specs 358-363.
## Dependencies And Historical Context
This package continues the current OperationRun truth line:
- **Spec 358 - OperationRun Queue Truth Foundation** established honest queued/running stale handling and explicitly deferred business-success reconciliation.
- **Spec 359 - OperationRun Reconciliation Adapter Framework & Review Compose Adapter** added a bounded adapter path and treated restore as an existing adapter precedent.
- **Spec 360 - OperationRun Canonical Cutover Cleanup** canonicalized the adapter registry, `context.reconciliation`, and dispatch/correlation semantics over the real restore and review-compose cases.
- **Spec 361 - Report and Evidence Reconciliation Adapters** added artifact-backed reconciliation for evidence snapshot and review-pack generation while keeping restore expansion out of scope.
- **Spec 362 - Sync, Capture, and Backup Operation Semantics** added selected-family proof semantics for sync, baseline capture, and backup schedule runs while deferring restore.
- **Spec 363 - Explicit UiActionContext Contract** is implemented action-context hardening and is dependency context only.
Restore-specific productization already exists and must not be reopened:
- **Spec 333 - Restore Create UX Final Productization** productized the pre-execution restore wizard.
- **Spec 335 - Restore Run Detail / Post-Execution Proof Productization** productized the restore detail proof surface.
- **Spec 181 - Restore Safety Integrity** defines earlier restore safety requirements and remains historical context.
Current repo truth already contains:
- `App\Support\Operations\Reconciliation\RestoreExecuteReconciliationAdapter`
- `App\Services\AdapterRunReconciler`
- `App\Services\OperationRunService`
- `App\Support\Operations\Reconciliation\ReconciliationResult`
- `App\Support\OperationRunOutcome` values `succeeded`, `partially_succeeded`, `blocked`, and `failed`
- `App\Support\RestoreRunStatus`
- `App\Jobs\ExecuteRestoreRunJob`
- `App\Listeners\SyncRestoreRunToOperationRun`
- restore proof/result presentation in `RestoreRunDetailPresenter`, `RestoreSafetyResolver`, and restore infolist Blade entries
The user draft's `verification_required` concept is valid product truth, but repo truth does not justify a new `OperationRun` outcome or new `restore.verify` operation type in this slice. Spec 364 represents verification gaps through existing outcomes plus restore-specific reason and evidence metadata.
## Spec Candidate Check *(mandatory - SPEC-GATE-001)*
- **Problem**: `restore.execute` can still be reconciled from terminal `RestoreRun` status in a way that risks overclaiming success for high-risk tenant-changing work. The current adapter maps `previewed` and `completed` to `succeeded` without requiring a complete proof bundle that distinguishes execution proof, provider acceptance, item-level result truth, verification evidence, scope safety, and audit continuity.
- **Today's failure**: A stale or interrupted restore operation can become a calm successful `OperationRun` when the related restore record is terminal, even if post-run evidence is unavailable, item outcomes are partial, provider proof is incomplete, or the restore status represents preview-only or pre-execution truth.
- **User-visible improvement**: Operators inspecting Operations or Restore Run detail will see restore execution truth that is honest by default: succeeded only when execution and proof are complete, partially succeeded when mutation occurred but verification or item truth is incomplete, blocked when safety gates prevent execution, and failed when no safe proof exists.
- **Smallest enterprise-capable version**: Harden exactly `restore.execute` reconciliation and visible fallout over current repo-real proof paths. Reuse existing `OperationRunOutcome`, `RestoreRunStatus`, `RestoreRun.results`, `RestoreRun.metadata`, `RestoreRun.operation_run_id`, audit links, and existing Restore Run detail proof presentation. Do not add new operation types, persistence, Graph contracts, or restore wizard behavior.
- **Explicit non-goals**: No new restore wizard, no new restore engine, no new rollback system, no `restore.preview` / `restore.validate` / `restore.verify` operation types, no new `OperationRun` outcome, no new table, no new Graph provider, no new Backup model, no new diff algorithm, no UI redesign of Operations or Restore detail, no retry console, no destructive cleanup action, no compatibility shims for pre-production historical rows.
- **Permanent complexity imported**: One bounded restore-proof decision path inside or beside the existing restore reconciliation adapter, focused Unit/Feature/Browser coverage, and small copy/metadata adjustments on existing Operations and Restore detail surfaces. No new persisted truth or cross-domain framework.
- **Why now**: Specs 358-362 intentionally matured OperationRun truth family by family. Restore is the highest-risk remaining adapter family because it can mutate real tenant configuration and false success is more dangerous than false calm for read-only report or backup work.
- **Why not local**: A Restore Run detail copy-only fix would not stop stale adapter reconciliation from writing overclaiming `OperationRun` truth. A job-only fix would not cover late/stale reconciliation. The proof boundary belongs in the shared adapter/service path and must remain visible in current run surfaces.
- **Approval class**: Core Enterprise.
- **Red flags triggered**: high-risk domain semantics, adapter proof hardening, restore-specific reason codes. **Defense**: this slice narrows the existing restore adapter rather than adding a framework; it uses existing outcomes, existing records, existing surfaces, and fail-closed rules.
- **Score**: Nutzen: 2 | Dringlichkeit: 2 | Scope: 2 | Komplexitaet: 1 | Produktnaehe: 2 | Wiederverwendung: 2 | **Gesamt: 11/12**
- **Decision**: approve.
## Candidate Source And Completed-Spec Guardrail
- **Candidate source**:
- direct user-provided Spec 364 draft in `pasted-text.txt`
- repo-real follow-through after Specs 358-363
- roadmap relationship: Golden Master Governance and restore safety/read-write separation; this is execution-truth hardening rather than a customer portal or productization backlog item
- **Queue boundary**: `docs/product/spec-candidates.md` records `no safe automatic next-best-prep target` for auto-selection. This package is an intentional manual promotion from direct user input, not an automatic backlog pick.
- **Completed-spec check result**:
- no `specs/364-*` package existed before this prep
- no local or remote `364-*` branch existed before this prep
- Specs 333, 335, and 358-363 are completed or implementation-context packages and must not be rewritten, normalized, unchecked, or reopened
- completed close-out, validation, smoke, browser, and checked task markers in related specs remain historical evidence
- **Close alternatives deferred**:
- broader restore workflow redesign is deferred because Spec 333 already covers create UX and Spec 335 already covers detail proof productization
- new `restore.verify` or rollback operation families are deferred until there is an explicit product/runtime source of truth for verification execution
- generic high-risk operation framework is deferred; `promotion.execute` and AI or operational-control high-risk flows need separate product decisions
- support-desk, customer-review, provider-scope, and canonical-link productization lanes are already specced/active/completed or manual-promotion only
- **Smallest viable implementation slice**: harden `restore.execute` adapter proof and related Operations/Restore detail presentation only.
## Summary
This feature prevents restore execution from being treated as successful merely because a related restore record reached a terminal status.
For `restore.execute`, the system must prove the intended mutation was accepted, item results are interpretable, verification or post-run evidence is available when success is claimed, scope remained safe, and audit continuity exists. If those signals are incomplete, the operation must finalize as partial, blocked, failed, or not reconciled instead of succeeding.
## Success Proof Bundle Matrix
The implementation must use this matrix as the narrow repo-real proof boundary before writing `OperationRunOutcome::Succeeded`.
| Proof element | Repo-real source to inspect | Success threshold | Missing or invalid proof result | Reason code guidance |
|---|---|---|---|---|
| Same-scope restore linkage | `OperationRun.context.restore_run_id`, `RestoreRun.operation_run_id`, `RestoreRun.workspace_id`, `RestoreRun.managed_environment_id`, default non-trashed `RestoreRun` query | A non-trashed RestoreRun belongs to the same workspace and managed environment as the OperationRun and links back through context or `operation_run_id` without contradiction | Return non-final `not_reconciled` when no same-scope RestoreRun can be safely identified; never use a wrong-scope or trashed RestoreRun as proof | `restore.proof_missing`, `restore.scope_mismatch`, `restore.run_deleted` |
| Execution proof | `RestoreRun.is_dry_run`, `RestoreRun.status`, `RestoreRun.started_at`, `RestoreRun.completed_at`, linked OperationRun status/outcome | Real execution, not preview/dry-run, reached an execution terminal status with timestamps consistent enough for audit and operator review | `previewed`, dry-run, missing start/completion proof, or terminal status alone cannot produce success | `restore.preview_only`, `restore.execution_proof_missing` |
| Provider acceptance / mutation proof | Existing restore service output persisted in `RestoreRun.results`, `RestoreRun.failure_reason`, and safe aggregate metadata | Results or metadata show the provider accepted or safely attempted the requested mutation and no provider-level rejection remains | Missing or rejected provider proof with a same-scope RestoreRun finalizes as `failed` or `blocked`; raw provider payloads stay out of reconciliation metadata | `restore.provider_proof_missing`, `restore.provider_rejected` |
| Item or aggregate result truth | `RestoreRun.results.items`, `RestoreRun.results.assignment_outcomes`, `RestoreRun.metadata.total_items`, `processed_items`, `succeeded_items`, `failed_items`, `skipped_items`, plus existing RestoreRun item helpers | Item or aggregate counts are interpretable, flat numeric where written to OperationRun, and show all required work succeeded | Mixed results produce `partially_succeeded`; absent result truth after execution withholds success | `restore.results_mixed`, `restore.results_missing` |
| Post-run evidence or explicit proof availability | Existing Restore Run detail evidence path, linked `OperationRun`, existing `EvidenceSnapshot` context when available, and safe metadata flags already persisted by the restore flow | Evidence is available or the existing restore result explicitly proves why recovery proof is available without a new persisted verification model | Missing evidence after mutation produces existing partial/blocked/failed truth with a visible proof-gap reason, not success | `restore.verification_required`, `restore.evidence_missing` |
| Audit continuity | `AuditLog` rows in the same workspace and managed environment, preferably linked through `operation_run_id` or stable restore action metadata; existing operation terminal audit remains service-owned | Same-scope audit trail can explain start/failure/completion or reconciliation without exposing secrets | Otherwise-complete proof without audit continuity must withhold success and record a safe proof-gap reason | `restore.audit_missing` |
`not_reconciled` is a `ReconciliationResult` decision, not an `OperationRunOutcome`. It is valid only when the adapter cannot safely identify enough same-scope restore proof to finalize the run. If a same-scope RestoreRun exists and proves execution reached an unsafe, partial, blocked, failed, or verification-gap state, the adapter must finalize with an existing outcome instead of using `not_reconciled` to hide operator-visible truth.
## Business/Product Value
- Reduces the risk of false recovery claims after tenant-changing operations.
- Makes restore monitoring and restore detail consistent with TenantPilot's read/write separation and audit-first posture.
- Keeps the platform sellable as Governance-of-Record by making high-risk mutation truth stricter than report, evidence, sync, or backup truth.
## Primary Users / Operators
- Tenant/MSP operators who start and review restore execution.
- Workspace owners/managers who approve or supervise high-risk changes.
- Support/platform operators who troubleshoot restore outcomes through Operations and audit evidence.
## Roadmap Relationship
Spec 364 belongs to the OperationRun execution-truth maturity line and the restore safety lane. It is not a new customer-facing workspace, not a new restore product surface, and not a generic high-risk operation framework.
## Spec Scope Fields *(mandatory)*
- **Scope**: canonical-view plus environment-bound restore execution truth
- **Primary Routes**:
- `/admin/workspaces/{workspace}/operations`
- `/admin/workspaces/{workspace}/operations/{run}`
- existing environment-scoped Restore Run list/create/detail routes via `App\Filament\Resources\RestoreRunResource`
- **Data Ownership**:
- `operation_runs` remain the only execution and reconciliation truth
- `restore_runs` remain the restore request/result truth
- `audit_logs` remain audit trail truth
- existing evidence snapshot links remain optional post-run evidence context only when already repo-backed
- no new persistence is introduced
- **RBAC**:
- existing workspace-first `OperationRun` access remains authoritative
- existing Restore Run policy/resource access remains authoritative
- non-members and wrong-scope actors remain `404`
- members missing restore capability remain `403` for restore execution
- no new capability strings are introduced
For canonical-view specs:
- **Default filter behavior when tenant-context is active**: the Operations hub remains workspace-scoped with explicit environment filters. Reconciliation must use the run's stored workspace and managed environment scope, not remembered environment state, current page filters, or Filament tenant fallback.
- **Explicit entitlement checks preventing cross-tenant leakage**: no adapter may reconcile to a restore run outside the operation's workspace and managed environment, and no related link may bypass current scope-safe routes.
## UI Surface Impact *(mandatory - UI-COV-001)*
- [ ] No UI surface impact
- [x] Existing page changed
- [ ] New page/route added
- [ ] Navigation changed
- [ ] Filament panel/provider surface changed
- [ ] New modal/drawer/wizard/action added
- [ ] New table/form/state added
- [ ] Customer-facing surface changed
- [x] Dangerous action changed
- [x] Status/evidence/review presentation changed
- [ ] Workspace/environment context presentation changed
## UI/Productization Coverage *(mandatory when UI Surface Impact is not "No UI surface impact")*
- **Route/page/surface**:
- `App\Filament\Pages\Monitoring\Operations`
- `App\Filament\Pages\Operations\TenantlessOperationRunViewer`
- `App\Filament\Resources\OperationRunResource` as shared implementation seam
- `App\Filament\Resources\RestoreRunResource` detail proof surface
- restore execution start/confirmation path only where proof or queued feedback reflects `restore.execute`
- **Current or new page archetype**: existing Operations monitoring/detail family plus existing environment-bound dangerous restore workflow/detail surfaces
- **Design depth**: Domain Pattern Surface / Manual Review Required for restore proof and dangerous-action truth
- **Repo-truth level**: repo-verified
- **Existing pattern reused**: current OperationRun monitoring family, current Restore Run detail proof model, current Restore Create safety/proof model, current `OperationRunLinks`, current BADGE-001 status badge semantics
- **New pattern required**: none; the change narrows existing restore proof/reconciliation behavior
- **Screenshot required**: one bounded browser smoke screenshot only if implementation materially changes visible hierarchy; otherwise existing Spec 333/335 screenshot anchors remain sufficient
- **Page audit required**: no new page-report identity unless implementation introduces a materially new visible hierarchy
- **Customer-safe review required**: no customer-facing surface; copy must still avoid false recovery claims
- **Dangerous-action review required**: yes; success wording and execute/verification claims must not overstate tenant recovery
- **Coverage files updated or explicitly not needed**:
- [ ] `docs/ui-ux-enterprise-audit/route-inventory.md`
- [ ] `docs/ui-ux-enterprise-audit/design-coverage-matrix.md`
- [ ] `docs/ui-ux-enterprise-audit/page-reports/...`
- [ ] `docs/ui-ux-enterprise-audit/strategic-surfaces.md`
- [ ] `docs/ui-ux-enterprise-audit/grouped-follow-up-candidates.md`
- [ ] `docs/ui-ux-enterprise-audit/unresolved-pages.md`
- [x] `N/A - existing Operations and Restore Run page families already cover these reachable surfaces unless implementation proves visible hierarchy drift`
- **No-impact rationale when applicable**: N/A
## Cross-Cutting / Shared Pattern Reuse *(mandatory)*
- **Cross-cutting feature?**: yes
- **Interaction class(es)**: status messaging, action links, dangerous-action proof wording, OperationRun reconciliation diagnostics, restore result/proof viewers
- **Systems touched**:
- `OperationRunReconciliationRegistry`
- `RestoreExecuteReconciliationAdapter`
- `AdapterRunReconciler`
- `OperationRunService`
- `ReconciliationResult`
- `RestoreSafetyResolver`
- `RestoreRunDetailPresenter`
- current Operations and Restore detail renderers
- **Existing pattern(s) to extend**: current adapter reconciliation path, current OperationRun lifecycle service ownership, current restore proof/detail model, current audit trail
- **Shared contract / presenter / builder / renderer to reuse**: `OperationRunService::applyReconciliationResult()`, `OperationRun::reconciliation()`, `OperationRunLinks`, `RestoreRunDetailPresenter`, `RestoreSafetyResolver`, `BadgeCatalog` / `BadgeRenderer`
- **Why the existing shared path is sufficient or insufficient**: the shared path exists, but the restore adapter's proof bar is too weak for tenant-changing work. It needs stricter restore-specific decision rules, not a new framework.
- **Allowed deviation and why**: a small restore proof evaluator is allowed only if it keeps adapter logic reviewable and stays derived-only over existing records.
- **Consistency impact**: restore success, partial, blocked, failed, and verification-gap wording must match across Operations, run detail, restore detail, notifications where existing, and audit-safe metadata.
- **Review focus**: no new outcome family, no success from `previewed`, no success from terminal status alone, no raw provider payload in default UI, no bypass of policies or `GraphClientInterface`.
## OperationRun UX Impact *(mandatory)*
- **Touches OperationRun start/completion/link UX?**: yes, completion/reconciliation and link presentation only
- **Shared OperationRun UX contract/layer reused**: `OperationRunService`, `OperationRunLinks`, `OperationUxPresenter`, current Operations hub/detail surfaces
- **Delegated start/completion UX behaviors**:
- existing restore queued feedback and run links remain on the shared path
- reconciliation finalization remains service-owned
- terminal notifications remain on the current central lifecycle path
- **Local surface-owned behavior that remains**: restore initiation inputs, preview/dry-run controls, confirmation copy, and restore-specific proof detail
- **Queued DB-notification policy**: unchanged; no new queued DB notification policy
- **Terminal notification path**: unchanged central lifecycle mechanism
- **Exception required?**: none
## Provider Boundary / Platform Core Check *(mandatory)*
- **Shared provider/platform boundary touched?**: yes
- **Boundary classification**: mixed
- **Seams affected**: provider-backed `restore.execute`, write gate, provider-operation start checks, restore result metadata, OperationRun reconciliation metadata
- **Neutral platform terms preserved or introduced**: `operation`, `execution proof`, `provider acceptance`, `verification evidence`, `scope safety`, `audit trail`, `managed environment`
- **Provider-specific semantics retained and why**: Microsoft/Intune restore behavior remains provider-owned because the current runtime has only Microsoft restore execution. Provider-specific payloads stay inside existing restore/provider services and are not promoted to platform-core taxonomy.
- **Why this does not deepen provider coupling accidentally**: the spec tightens proof criteria around existing `restore.execute`; it does not create provider-neutral restore abstractions, provider registries, or Graph contract expansion.
- **Follow-up path**: `follow-up-spec` only if future restore verification becomes a distinct queued operation with repo-real execution and artifact truth.
## UI / Surface Guardrail Impact *(mandatory)*
| Surface / Change | Operator-facing surface change? | Native vs Custom | Shared-Family Relevance | State Layers Touched | Exception Needed? | Low-Impact / `N/A` Note |
|---|---|---|---|---|---|---|
| Operations hub restore outcome wording | yes | Native Filament page | shared monitoring family | page, table row | no | existing surface only |
| Tenantless run detail restore reconciliation explanation | yes | Native Filament page | shared monitoring detail family | detail | no | explanation and proof metadata only |
| Restore Run detail proof state | yes | Filament infolist plus existing custom Blade entry | restore proof/detail family | detail | no | proof-safe presentation over existing state |
| Restore execute confirmation/start feedback | yes | Filament action/wizard | dangerous action family | wizard/action | no | no new action; proof semantics may be tightened |
## Decision-First Surface Role *(mandatory)*
| Surface | Decision Role | Human-in-the-loop Moment | Immediately Visible for First Decision | On-Demand Detail / Evidence | Why This Is Primary or Why Not | Workflow Alignment | Attention-load Reduction |
|---|---|---|---|---|---|---|---|
| Operations hub | Primary Decision Surface | Decide whether a restore run needs follow-up | lifecycle, outcome, proof gap, one safe next action | full run detail, restore detail, diagnostics | primary because it is the canonical monitoring queue | aligns with operations triage | removes false-success row reading |
| Tenantless run detail | Tertiary Evidence / Diagnostics Surface | Confirm why restore reconciliation finalized a run | one restore-specific explanation and related restore link | raw context and support diagnostics | tertiary because the run is selected | preserves current detail role | keeps proof reason above raw context |
| Restore Run detail | Primary Decision Surface | Decide whether recovery proof is available or follow-up is required | result state, reason, impact, proof availability, one primary next action | item outcomes, raw result payload, evidence diagnostics | primary for restore result truth | follows post-execution restore review | separates completion from recovery proof |
| Restore execute confirmation | Primary Decision Surface | Decide whether real tenant mutation may start | safety gates, preview/check currentness, mutation scope, confirmation | preview details and diagnostics | primary because mutation can alter tenant configuration | follows safe restore execution flow | prevents action before proof review |
## Audience-Aware Disclosure *(mandatory)*
| Surface | Audience Modes In Scope | Decision-First Default-Visible Content | Operator Diagnostics | Support / Raw Evidence | One Dominant Next Action | Hidden / Gated By Default | Duplicate-Truth Prevention |
|---|---|---|---|---|---|---|---|
| Operations hub | operator-MSP, support-platform | outcome, proof gap, related restore target | reconciliation reason, summary counts | raw context only in run detail | open restore run or inspect run | raw provider payloads | one row outcome plus one link |
| Run detail | operator-MSP, support-platform | restore-specific reconciliation explanation | context.reconciliation and related records | raw context and failures secondary | open restore run / inspect proof | raw provider payloads and IDs | explanation references one proof source |
| Restore detail | operator-MSP, support-platform | recovery proof question, result summary, evidence state | item outcomes, failure family, audit links | raw results collapsed | open operation proof / open evidence / review gap | raw JSON, internal reason ownership | presenter owns result decision once |
## UI/UX Surface Classification *(mandatory)*
| Surface | Action Surface Class | Surface Type | Likely Next Operator Action | Primary Inspect/Open Model | Row Click | Secondary Actions Placement | Destructive Actions Placement | Canonical Collection Route | Canonical Detail Route | Scope Signals | Canonical Noun | Critical Truth Visible by Default | Exception Type / Justification |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Operations hub | List / Workbench | Monitoring queue | inspect a restore needing follow-up | row/detail route | allowed | row/detail secondary links | none introduced | `/admin/workspaces/{workspace}/operations` | `/admin/workspaces/{workspace}/operations/{run}` | workspace and environment | Operations / Operation | restore outcome and proof gap | none |
| Restore Run detail | Detail / Evidence | Dangerous workflow result | review restore result proof | detail page | N/A | diagnostics and related links after summary | none introduced | Restore Runs list | Restore Run detail | workspace and environment | Restore Run | completion vs recovery proof | none |
| Restore execute confirmation | Workflow / Dangerous Action | Restore execution gate | confirm or stop restore | wizard step | N/A | proof/diagnostics panels | final confirm step only | Restore Runs list | Restore Run detail after creation | workspace, environment, mutation scope | Restore Run | safety and proof readiness | none |
## Operator Surface Contract *(mandatory)*
| Surface | Primary Persona | Decision / Operator Action Supported | Surface Type | Primary Operator Question | Default-visible Information | Diagnostics-only Information | Status Dimensions Used | Mutation Scope | Primary Actions | Dangerous Actions |
|---|---|---|---|---|---|---|---|---|---|---|
| Restore Run detail | Tenant operator / MSP operator | Decide whether recovery proof is available or follow-up is required | Restore result detail | Was this restore executed safely, and is recovery proof available? | result state, reason, impact, operation proof, evidence state, summary counts | item JSON, raw provider diagnostics, raw context | execution outcome, provider acceptance, verification evidence, recovery proof, lifecycle | read-only detail over a prior Microsoft tenant mutation | open operation proof, open evidence, review proof gap | none introduced |
| Operations run detail | Workspace operator / support operator | Inspect restore-linked operation proof | Operation diagnostics detail | Why did this restore operation finish this way? | lifecycle, outcome, restore reconciliation reason, related restore link | raw run context, failures, support evidence | lifecycle, execution outcome, reconciliation proof | read-only monitoring | open restore run | none introduced |
| Restore execute confirmation | Tenant operator / MSP operator | Confirm whether real restore execution may start | Dangerous workflow wizard | Can this restore mutate the Microsoft tenant now? | safety gates, preview/check currentness, mutation scope, typed confirmation, proof limits | preview detail, mapping detail, raw diff | readiness, mutation scope, evidence availability | Microsoft tenant when execution proceeds | execute restore after confirmation | execute restore |
## Proportionality Review *(mandatory when structural complexity is introduced)*
- **New source of truth?**: no
- **New persisted entity/table/artifact?**: no
- **New abstraction?**: maybe; a small restore proof evaluator may be introduced only if it replaces duplicated adapter/detail proof logic and stays local to restore execution truth
- **New enum/state/reason family?**: no new `OperationRun` outcome or persisted status family; small restore-specific reason codes such as `restore.verification_required` may be derived metadata only if they change operator next action
- **New cross-domain UI framework/taxonomy?**: no
- **Current operator problem**: false successful reconciliation for tenant-changing restore execution can make operators believe recovery is proven when only terminal status or partial execution exists.
- **Existing structure is insufficient because**: the current restore adapter maps terminal restore status directly to OperationRun outcome without a strict proof bundle; restore detail proof surfaces already distinguish evidence but the run lifecycle can still overclaim.
- **Narrowest correct implementation**: harden the existing `RestoreExecuteReconciliationAdapter` and existing presenters to require proof for success and fail closed otherwise.
- **Ownership cost**: a small set of restore proof rules and focused tests that future restore changes must honor.
- **Alternative intentionally rejected**: adding `verification_required` as a new OperationRun outcome or building a new restore verification operation family is rejected because the current repo has no corresponding execution truth.
- **Release truth**: current-release truth; restore execution exists and is high-risk now.
### Compatibility Posture
This feature assumes a pre-production environment.
Backward compatibility, legacy aliases, migration shims, historical fixtures, and compatibility-specific tests are out of scope unless explicitly required by this spec.
Canonical replacement is preferred over preservation. Existing `RestoreRunStatus::Aborted` and `RestoreRunStatus::CompletedWithErrors` may remain as current housekeeping semantics, but Spec 364 must not add new compatibility-only restore status aliases.
## Testing / Lane / Runtime Impact *(mandatory for runtime behavior changes)*
- **Test purpose / classification**: Unit + Feature/Livewire; Browser only if visible hierarchy changes
- **Validation lane(s)**: fast-feedback + confidence; browser only if visible hierarchy changes; PostgreSQL only if implementation touches query/index/lock behavior, which is not expected
- **Why this classification and these lanes are sufficient**: Unit tests prove proof mapping and fail-closed adapter decisions; Feature tests prove Operations/Restore detail and authorization-safe fallout; one Browser smoke is justified only for changed high-risk visible proof hierarchy.
- **New or expanded test families**: focused Spec 364 restore reconciliation tests; no heavy-governance family
- **Fixture / helper cost impact**: reuse existing restore, backup set, operation run, evidence, and workspace fixtures; do not widen defaults
- **Heavy-family visibility / justification**: no heavy-governance family; browser smoke only if visible hierarchy changes
- **Special surface test profile**: shared-detail-family + monitoring-state-page + dangerous-workflow
- **Standard-native relief or required special coverage**: not standard relief; restore is high-risk and needs focused proof
- **Reviewer handoff**: reviewers must verify no success outcome is produced from preview-only, incomplete, wrong-scope, or missing-verification restore truth
- **Budget / baseline / trend impact**: low; bounded Unit/Feature tests and optional one browser smoke
- **Escalation needed**: document-in-feature
- **Active feature PR close-out entry**: Guardrail / Exception / Smoke Coverage
- **Planned validation commands**:
- `cd apps/platform && ./vendor/bin/sail artisan test --compact --filter=Spec364`
- `cd apps/platform && ./vendor/bin/sail artisan test --compact --filter=RestoreRun`
- `cd apps/platform && ./vendor/bin/sail artisan test --compact --filter=OperationRun`
- `cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Browser/Spec364RestoreHighRiskOperationReconciliationSmokeTest.php --compact` if browser coverage is added
- `cd apps/platform && ./vendor/bin/sail pint --dirty`
## User Scenarios & Testing *(mandatory)*
### User Story 1 - Reconcile Restore Success Only With Complete Proof (Priority: P1)
As an MSP operator reviewing a restore-linked operation, I want `restore.execute` to become successful only when execution proof, provider acceptance, item result truth, post-run evidence, and audit continuity are present, so that I do not mistake a terminal record for verified recovery.
**Why this priority**: This prevents the most dangerous false success claim in a tenant-changing flow.
**Independent Test**: Create restore runs with completed, previewed, partial, failed, and incomplete-proof states; run adapter reconciliation; verify only complete proof produces `succeeded`.
**Acceptance Scenarios**:
1. **Given** a `restore.execute` OperationRun linked to a completed RestoreRun with execution proof, provider acceptance, item counts, and post-run evidence, **When** adapter reconciliation runs, **Then** the OperationRun is completed with `succeeded` and proof metadata is stored in `context.reconciliation`.
2. **Given** a linked RestoreRun is only `previewed`, **When** adapter reconciliation runs, **Then** the OperationRun is not marked as successful execution.
3. **Given** provider acceptance or item result proof is missing, **When** reconciliation runs, **Then** success is withheld and the decision becomes partial, failed, blocked, or not reconciled according to the available proof.
---
### User Story 2 - Surface Partial And Verification-Gap Truth Without New Outcomes (Priority: P1)
As an operator reviewing a restore result, I want verification gaps and mixed item outcomes to be visible without creating a misleading new run state, so that I know the next safe action.
**Why this priority**: Verification gaps are real operator truth, but a new persisted outcome would create avoidable platform complexity.
**Independent Test**: Reconcile completed-but-unverified and mixed-outcome restore runs and assert existing `partially_succeeded`, `blocked`, or `failed` outcomes plus restore-specific reason metadata and visible copy.
**Acceptance Scenarios**:
1. **Given** a restore mutates tenant state but post-run evidence is unavailable, **When** reconciliation finalizes, **Then** the OperationRun outcome is not `succeeded`; it carries a restore verification-gap reason and a primary next action to review or generate evidence.
2. **Given** some restore items succeed and others fail or are skipped, **When** reconciliation finalizes, **Then** the outcome is `partially_succeeded` and summary counts remain flat numeric values.
3. **Given** a write gate, provider capability, backup availability, or scope safety blocker prevents meaningful execution, **When** reconciliation finalizes or fails closed, **Then** the outcome is `blocked` or `failed` with safe reason metadata.
---
### User Story 3 - Preserve Scope Safety And Audit Continuity (Priority: P2)
As a platform/support operator, I want restore reconciliation to prove it matched the correct workspace and managed environment and to preserve audit references, so that troubleshooting does not expose or conflate tenant data.
**Why this priority**: Restore proof is only trustworthy if it is scope-safe and audit-backed.
**Independent Test**: Attempt reconciliation across wrong workspace/environment restore records and verify no reconciliation occurs; verify same-scope runs include safe audit/proof identifiers only.
**Acceptance Scenarios**:
1. **Given** a restore run from another managed environment has the same ID-like context shape, **When** reconciliation evaluates the OperationRun, **Then** the adapter refuses to reconcile it.
2. **Given** audit continuity is missing for a high-risk restore execution, **When** success would otherwise be possible, **Then** success is withheld or an explicit proof-gap reason is recorded.
3. **Given** a user lacks access to the related restore detail, **When** Operations renders, **Then** no hidden restore metadata or tenant existence leaks.
---
### User Story 4 - Keep Unsupported High-Risk Restore Families Out Of Scope (Priority: P3)
As a reviewer, I want the spec to explicitly reject new restore operation families and generic high-risk operation machinery, so that implementation stays bounded.
**Why this priority**: The user draft contains valid future language, but widening this slice would collide with the constitution's anti-bloat rules.
**Independent Test**: Static or feature assertions prove only `restore.execute` is registered for Spec 364 restore reconciliation hardening and no `restore.verify`, `restore.rollback.*`, or generic high-risk registry is introduced.
**Acceptance Scenarios**:
1. **Given** a run type such as `restore.verify` or `restore.rollback.execute`, **When** Spec 364 reconciliation support is inspected, **Then** it is unsupported unless a future spec creates repo-real execution truth for it.
2. **Given** `promotion.execute` or AI execution is high-risk, **When** this implementation is reviewed, **Then** it remains out of scope and no generic high-risk framework appears.
### Edge Cases
- A `RestoreRun` status is `previewed`; this is pre-execution truth and must not mark `restore.execute` as successful.
- A `RestoreRun` is `completed` but item outcomes or summary counts are absent; success must be withheld unless proof is sufficient.
- A restore job writes a terminal failure after a provider exception; failure reason must be sanitized and no raw provider payload may appear in default UI or audit metadata.
- A restore is blocked by write gate or provider capability; no new execution success may be inferred from a related existing record.
- A restore has post-run evidence available but it belongs to a different workspace or managed environment; reconciliation must fail closed.
- A system-run or initiator-null restore context must follow existing OperationRun notification rules and avoid initiator-only terminal DB notifications.
## Requirements *(mandatory)*
### Functional Requirements
- **FR-364-001**: The system MUST support Spec 364 hardening only for canonical `restore.execute` in this slice.
- **FR-364-002**: The system MUST NOT mark `restore.execute` as `succeeded` from `RestoreRunStatus::Previewed`.
- **FR-364-003**: The system MUST NOT mark `restore.execute` as `succeeded` from terminal RestoreRun status alone.
- **FR-364-004**: The system MUST require a complete success proof bundle before writing `OperationRunOutcome::Succeeded` for restored execution.
- **FR-364-005**: The success proof bundle MUST follow the Success Proof Bundle Matrix and include same-scope RestoreRun linkage, execution proof, provider acceptance or equivalent safe mutation proof, interpretable item or aggregate result truth, post-run evidence or explicit proof availability, and audit continuity.
- **FR-364-006**: Missing verification or post-run evidence after mutation MUST NOT produce `succeeded`; it MUST produce an existing outcome such as `partially_succeeded`, `blocked`, or `failed` with restore-specific reason metadata.
- **FR-364-007**: Mixed item results MUST produce `partially_succeeded` and flat numeric summary counts where counts are available.
- **FR-364-008**: Write-gate, provider capability, backup availability, or scope-safety blockers MUST produce `blocked` or `failed` when same-scope restore proof exists; they may produce a non-final `not_reconciled` decision only when the adapter cannot safely identify same-scope proof.
- **FR-364-009**: Reconciliation MUST fail closed when the linked RestoreRun is missing, wrong-scope, soft-deleted in a way that invalidates proof, lacks required proof metadata, or lacks required audit continuity.
- **FR-364-010**: Reconciliation metadata MUST be safe for audit and operator display: no secrets, no raw provider payloads, no raw credential payloads, and no hidden tenant hints.
- **FR-364-011**: The Operations hub and run detail MUST show restore-specific success, partial, blocked, failed, or proof-gap meaning using existing shared OperationRun presentation paths.
- **FR-364-012**: Restore Run detail MUST continue to distinguish operation proof from post-run evidence and MUST not claim verified recovery when evidence is absent.
- **FR-364-013**: Implementation MUST NOT introduce a new `OperationRunOutcome`, new `OperationRunStatus`, new persisted restore verification table, or new restore operation type.
- **FR-364-014**: Unsupported future restore or high-risk operation types MUST remain unsupported and fail closed unless a future spec provides repo-real execution truth.
- **FR-364-015**: Tests MUST prove wrong-workspace and wrong-managed-environment restore records cannot reconcile a run.
- **FR-364-016**: Tests MUST prove success, partial, blocked, failed, preview-only, missing-proof, missing-audit, soft-deleted RestoreRun, and wrong-scope branches.
### Non-Functional Requirements
- **NFR-364-001**: Reconciliation must remain DB-local and must not call Microsoft Graph or any provider API.
- **NFR-364-002**: Reconciliation must remain idempotent and service-owned through current `OperationRunService` paths.
- **NFR-364-003**: Default-visible UI must remain calm but not falsely reassuring.
- **NFR-364-004**: Summary counts must use existing flat numeric OperationRun summary rules.
- **NFR-364-005**: No migration, env var, scheduler, queue family, package, panel provider, or asset registration is expected.
### Key Entities *(include if feature involves data)*
- **OperationRun**: existing execution and reconciliation truth for `restore.execute`.
- **RestoreRun**: existing restore request/result truth with status, preview, results, metadata, and optional operation link.
- **AuditLog**: existing audit trail truth for restore started/failed/executed events.
- **EvidenceSnapshot**: optional post-run evidence context where existing links already prove scope-safe evidence.
## Success Criteria *(mandatory)*
### Measurable Outcomes
- **SC-364-001**: `restore.execute` reconciliation produces `succeeded` only for complete-proof fixtures and never for preview-only, missing-proof, missing-audit, soft-deleted, wrong-scope, or mixed-result fixtures.
- **SC-364-002**: Focused Spec 364 Unit/Feature tests cover all primary outcome branches and pass in the narrow validation lane.
- **SC-364-003**: Existing Operations and Restore detail surfaces present proof gaps without introducing a new route, page family, or customer-facing surface.
- **SC-364-004**: No new database table, migration, OperationRun outcome/status, restore operation type, package, or Graph contract is introduced.
- **SC-364-005**: Audit/proof metadata contains only safe identifiers, counts, reason codes, and links; no secrets or raw provider payloads appear.
## Assumptions
- Existing Restore Run result and metadata payloads already contain enough safe aggregate or item-level truth to distinguish success, partial, blocked, failed, and proof-gap cases; if implementation proves otherwise, success must be withheld rather than inventing a new persisted proof model.
- Existing post-run evidence links may be unavailable for many restores; this is a partial/proof-gap state, not a success state.
- Current pre-production posture allows canonical cleanup without historical compatibility shims.
## Risks
- **Risk 1 - Existing restore data lacks enough proof for success**: mitigate by failing closed and documenting exact missing proof rather than loosening success criteria.
- **Risk 2 - Over-widening into a new restore verification operation**: mitigate by forbidding new operation types in this spec and deferring verification execution to a future spec.
- **Risk 3 - UI repeats proof truth in multiple places**: mitigate by keeping Restore Run detail presenter and OperationRun presenter aligned and avoiding duplicate default-visible summaries.
- **Risk 4 - Test fixtures become broad and expensive**: mitigate by using focused factories and existing helpers without widening global defaults.
## Open Questions
No open question blocks preparation. Implementation must verify the exact available RestoreRun metadata keys before deciding whether any small derived helper is needed.
## Follow-Up Spec Candidates
- Restore verification operation family v1, only if a future product decision creates repo-real queued verification and evidence truth.
- Restore rollback execution truth, only after restore verification semantics exist.
- Cross-domain high-risk operation framework, only if at least two additional high-risk operation families need the same proof boundary and cannot be handled locally.
- Customer-safe restore recovery report, only after internal proof semantics are stable.

View File

@ -0,0 +1,126 @@
# Tasks: Spec 364 - Restore and High-Risk Operation Reconciliation
**Input**: Design documents from `/specs/364-restore-high-risk-operation-reconciliation/`
**Prerequisites**: `spec.md`, `plan.md`
**Tests**: Runtime behavior changes are required to use Pest. This feature needs Unit and Feature coverage. Browser smoke is required only if visible Operations or Restore detail hierarchy materially changes.
**Operations**: Existing `restore.execute` runs must continue to use `OperationRunService`, the current OperationRun UX contract, and current queued/terminal notification behavior. `OperationRun.status` and `OperationRun.outcome` transitions must remain service-owned.
**RBAC**: No new capability strings are expected. Existing workspace/managed-environment entitlement and restore execution capability semantics remain authoritative.
**Non-goals**: Do not add new operation types, new `OperationRun` outcomes, new migrations, new Graph contracts, new high-risk framework machinery, new Restore wizard behavior, or compatibility shims.
## Test Governance Checklist
- [x] Lane assignment is named and is the narrowest sufficient proof for restore proof hardening.
- [x] New or changed tests stay in the smallest honest family; any browser smoke is explicit and justified.
- [x] Shared helpers, factories, seeds, fixtures, and context defaults stay cheap by default.
- [x] Planned validation commands cover the change without pulling unrelated lane cost.
- [x] The declared surface profiles `shared-detail-family`, `monitoring-state-page`, and `dangerous-workflow` are explicit.
- [x] No new outcome/status family, persistence layer, or generic high-risk framework is introduced.
## Phase 1: Setup And Repo Truth
**Purpose**: Re-confirm the exact current restore and OperationRun truth before implementation.
- [x] T001 Re-read `specs/364-restore-high-risk-operation-reconciliation/spec.md`, `plan.md`, `tasks.md`, and `checklists/requirements.md`.
- [x] T002 Re-read related historical context only: Specs 333, 335, 358, 359, 360, 361, 362, and 363. Do not modify their artifacts.
- [x] T003 [P] Confirm current restore adapter behavior in `apps/platform/app/Support/Operations/Reconciliation/RestoreExecuteReconciliationAdapter.php`.
- [x] T004 [P] Confirm current service-owned reconciliation write behavior in `apps/platform/app/Services/OperationRunService.php` and `apps/platform/app/Services/AdapterRunReconciler.php`.
- [x] T005 [P] Confirm current restore result/proof truth in `apps/platform/app/Support/RestoreSafety/RestoreSafetyResolver.php`, `apps/platform/app/Filament/Resources/RestoreRunResource/Presenters/RestoreRunDetailPresenter.php`, and `apps/platform/resources/views/filament/infolists/entries/restore-results.blade.php`.
- [x] T006 [P] Confirm current restore execution start and audit behavior in `apps/platform/app/Filament/Resources/RestoreRunResource.php`, `apps/platform/app/Jobs/ExecuteRestoreRunJob.php`, and `apps/platform/app/Services/Intune/RestoreService.php`.
- [x] T007 Confirm no migration, package, env var, queue family, scheduler, storage, Filament panel/provider, asset registration, or global-search change is required.
## Phase 2: Foundational Tests
**Purpose**: Add failing tests for proof-gated `restore.execute` reconciliation before runtime edits.
- [x] T008 [P] Add coverage for complete success proof, preview-only, missing post-run evidence, mixed item outcomes, blocked proof, failed proof, missing audit continuity, soft-deleted RestoreRun proof, and wrong-scope proof. Completed in `apps/platform/tests/Feature/Operations/Spec364RestoreExecuteReconciliationTest.php` because the proof contract is DB-backed.
- [x] T009 [P] Add coverage asserting `RestoreExecuteReconciliationAdapter` does not return `succeeded` for `previewed`, terminal-status-only, missing-proof, missing-audit, soft-deleted, or wrong-scope RestoreRuns. Completed in the focused Spec 364 Feature coverage.
- [x] T010 [P] Add Feature coverage in `apps/platform/tests/Feature/Operations/Spec364RestoreExecuteReconciliationTest.php` proving adapter reconciliation writes safe `context.reconciliation`, flat numeric summary counts, safe audit/proof reason metadata, and existing outcomes only.
- [x] T011 [P] Add Feature coverage proving a RestoreRun from another workspace, another managed environment, or a soft-deleted RestoreRun cannot reconcile the OperationRun or leak related links. Completed in `Spec364RestoreExecuteReconciliationTest.php`.
- [x] T012 [P] Add or extend Feature coverage for current Operations/detail fallout so proof gaps read as partial/blocked/failed instead of calm success. Completed through focused Spec 364 coverage plus restore/operation regression suites.
- [x] T013 [P] Add or extend Restore detail Feature coverage so operation proof and post-run evidence remain distinct. Completed through existing Spec 335 restore detail coverage and updated recovery-attention regressions.
## Phase 3: Restore Adapter Hardening (P1)
**Goal**: Reconcile `restore.execute` as `succeeded` only with a complete proof bundle.
**Independent Test**: T008-T011 pass and show no success for preview-only, missing-proof, missing-audit, soft-deleted, wrong-scope, or mixed-result cases.
- [x] T014 Implement proof-gated success rules from the spec's Success Proof Bundle Matrix in `apps/platform/app/Support/Operations/Reconciliation/RestoreExecuteReconciliationAdapter.php`.
- [x] T015 Ensure `RestoreRunStatus::Previewed` is treated as pre-execution truth and never maps to successful `restore.execute` execution in `apps/platform/app/Support/Operations/Reconciliation/RestoreExecuteReconciliationAdapter.php`.
- [x] T016 Map mixed item outcomes or incomplete verification/post-run evidence to `OperationRunOutcome::PartiallySucceeded` with restore-specific reason metadata in `apps/platform/app/Support/Operations/Reconciliation/RestoreExecuteReconciliationAdapter.php`.
- [x] T017 Map write-gate/provider/backup/scope blockers to existing `blocked` or `failed` outcomes when same-scope restore proof exists, and reserve non-final `not_reconciled` only for cases where the adapter cannot safely identify same-scope proof.
- [x] T018 Keep summary counts flat numeric-only and compatible with current OperationRun summary rules in `apps/platform/app/Support/Operations/Reconciliation/RestoreExecuteReconciliationAdapter.php`.
- [x] T019 N/A - proof-rule duplication stayed bounded inside `RestoreExecuteReconciliationAdapter`; no broader helper or framework was added.
- [x] T020 Verify `apps/platform/app/Services/AdapterRunReconciler.php` still syncs restore timestamps only from safe evidence and does not turn non-final decisions into writes.
## Phase 4: Visible Proof Fallout (P1/P2)
**Goal**: Keep Operations and Restore detail honest without creating a new UI pattern.
**Independent Test**: Feature tests show Operations and Restore detail distinguish success, partial, blocked, failed, and proof gaps.
- [x] T021 N/A - current OperationRun surfaces already render `context.reconciliation` proof-gap reasons.
- [x] T022 Update `apps/platform/app/Filament/Resources/OperationRunResource.php` only if the shared implementation seam must expose restore proof-gap metadata consistently.
- [x] T023 N/A - the presenter already preserved operation proof versus post-run evidence; only recovery-attention outcome interpretation needed adjustment.
- [x] T024 Preserved the Restore Run detail distinction between operation proof and post-run evidence in `apps/platform/resources/views/filament/infolists/entries/restore-results.blade.php`.
- [x] T025 Keep raw provider payloads, raw credential data, and internal reason ownership out of default-visible UI and audit metadata.
- [x] T026 Do not add a new page, route, navigation entry, Filament panel/provider registration, or asset registration.
## Phase 5: Unsupported Families And Anti-Creep
**Goal**: Keep Spec 364 bounded to `restore.execute`.
- [x] T027 Add or extend static/Unit coverage proving `restore.verify`, `restore.rollback.*`, `promotion.execute`, AI execution, and generic high-risk operation types are not registered by Spec 364.
- [x] T028 Verify `apps/platform/app/Support/Operations/Reconciliation/OperationRunReconciliationRegistry.php` does not gain a generic high-risk adapter or future restore family in this slice.
- [x] T029 Verify no new `OperationRunOutcome`, `OperationRunStatus`, `RestoreRunStatus`, database migration, or persisted restore verification model was introduced.
- [x] T030 Record any future restore verification or rollback needs as follow-up only in the implementation close-out note, not in runtime code.
## Phase 6: Validation And Close-Out
**Purpose**: Run focused validation and document the final proof boundary.
- [x] T031 Run `cd apps/platform && php vendor/bin/pest --filter=Spec364`.
- [x] T032 Run `cd apps/platform && php vendor/bin/pest --filter=Restore`.
- [x] T033 Run `cd apps/platform && php vendor/bin/pest --filter=OperationRun`.
- [x] T034 N/A - no new Spec 364 browser smoke file was added; existing restore browser smoke coverage ran under the broad Restore filter.
- [x] T035 Run `cd apps/platform && php vendor/bin/pint --dirty`.
- [x] T036 Run `git diff --check`.
- [x] T037 Confirm final implementation keeps reconciliation DB-local and does not introduce Graph/provider calls from adapter, presenter, or render paths.
- [x] T038 Confirm Livewire v4.0+ compliance remains unchanged and Filament providers remain registered through `apps/platform/bootstrap/providers.php`.
- [x] T039 Confirm globally searchable resources were not enabled or changed; if any resource global search was touched unexpectedly, document View/Edit page safety or disable it.
- [x] T040 Confirm destructive/high-impact restore actions still execute via `->action(...)`, require confirmation, reauthorize server-side, write audit logs, and are covered by tests.
- [x] T041 Confirmed no new Filament assets were registered; `filament:assets` deployment handling is unchanged.
- [x] T042 Close-out is recorded in the agent final response: proof boundary, tests run, browser smoke decision, deployment impact, and deferred follow-up candidates.
## Dependencies & Execution Order
- Phase 1 must finish before tests and implementation.
- Phase 2 tests should be added before Phase 3 runtime edits.
- Phase 3 blocks Phase 4 visible fallout because UI should consume final proof semantics.
- Phase 5 can run after Phase 3 and before close-out.
- Phase 6 is final validation.
## Parallel Opportunities
- T003-T006 can run in parallel.
- T008-T013 can be authored in parallel if they touch separate test files.
- T021-T024 can be split only after proof semantics are final.
- Validation commands should run after implementation and formatting.
## Implementation Strategy
1. Prove the current adapter overclaims success with focused failing tests.
2. Tighten adapter proof rules without changing persistence or operation outcomes.
3. Adjust visible presentation only where existing copy cannot express stricter proof truth.
4. Guard unsupported future families.
5. Run focused validation and browser smoke only if UI hierarchy changed.
## Notes
- `[P]` tasks are parallelizable only when they touch separate files.
- Every user-visible proof or reason label must avoid false recovery claims.
- Do not implement application code during SpecKit preparation.