Implemented restore high risk operation reconciliation. Co-authored-by: Ahmed Darrazi <ahmed.darrazi@live.de> Reviewed-on: #435
308 lines
12 KiB
PHP
308 lines
12 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use App\Filament\Resources\OperationRunResource;
|
|
use App\Models\AuditLog;
|
|
use App\Models\BackupSet;
|
|
use App\Models\EvidenceSnapshot;
|
|
use App\Models\ManagedEnvironment;
|
|
use App\Models\OperationRun;
|
|
use App\Models\RestoreRun;
|
|
use App\Services\AdapterRunReconciler;
|
|
use App\Support\Evidence\EvidenceCompletenessState;
|
|
use App\Support\Evidence\EvidenceSnapshotStatus;
|
|
use App\Support\OperationRunLinks;
|
|
use App\Support\OperationRunOutcome;
|
|
use App\Support\OperationRunStatus;
|
|
use App\Support\Operations\Reconciliation\RestoreExecuteReconciliationAdapter;
|
|
use App\Support\RestoreRunStatus;
|
|
|
|
it('requires result proof, active evidence, and audit continuity before succeeding restore execute runs in Spec364', function (): void {
|
|
[, $tenant, $run, $restoreRun] = spec364RestoreFixture();
|
|
|
|
$evidenceSnapshot = spec364EvidenceSnapshotFor($tenant, $run);
|
|
$auditLog = spec364RestoreAuditFor($tenant, $run, $restoreRun);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('reconciled_succeeded')
|
|
->and($result?->outcome)->toBe(OperationRunOutcome::Succeeded->value)
|
|
->and($result?->reasonCode)->toBe('restore.proof_complete')
|
|
->and($result?->summaryCounts)->toMatchArray([
|
|
'total' => 1,
|
|
'processed' => 1,
|
|
'succeeded' => 1,
|
|
'failed' => 0,
|
|
'skipped' => 0,
|
|
])
|
|
->and($result?->evidence['evidence_snapshot_id'] ?? null)->toBe((int) $evidenceSnapshot->getKey())
|
|
->and($result?->evidence['audit_log_id'] ?? null)->toBe((int) $auditLog->getKey());
|
|
});
|
|
|
|
it('marks preview-only restore truth as blocked instead of succeeded in Spec364', function (): void {
|
|
[, , $run] = spec364RestoreFixture([
|
|
'status' => RestoreRunStatus::Previewed->value,
|
|
'is_dry_run' => true,
|
|
'results' => [],
|
|
'metadata' => [],
|
|
'completed_at' => null,
|
|
]);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('blocked')
|
|
->and($result?->outcome)->toBe(OperationRunOutcome::Blocked->value)
|
|
->and($result?->reasonCode)->toBe('restore.preview_only');
|
|
});
|
|
|
|
it('marks mixed restore results partially succeeded without requiring recovery evidence first in Spec364', function (): void {
|
|
[, , $run] = spec364RestoreFixture([
|
|
'results' => [
|
|
'foundations' => [],
|
|
'items' => [
|
|
['status' => 'partial', 'policy_identifier' => 'policy-partial'],
|
|
],
|
|
],
|
|
'metadata' => [
|
|
'total' => 1,
|
|
'succeeded' => 0,
|
|
'failed' => 0,
|
|
'skipped' => 0,
|
|
'partial' => 1,
|
|
'non_applied' => 0,
|
|
],
|
|
]);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('reconciled_partially_succeeded')
|
|
->and($result?->outcome)->toBe(OperationRunOutcome::PartiallySucceeded->value)
|
|
->and($result?->reasonCode)->toBe('restore.results_mixed');
|
|
});
|
|
|
|
it('requires post-run evidence before claiming successful restore recovery in Spec364', function (): void {
|
|
[, $tenant, $run, $restoreRun] = spec364RestoreFixture();
|
|
|
|
spec364RestoreAuditFor($tenant, $run, $restoreRun);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('reconciled_partially_succeeded')
|
|
->and($result?->outcome)->toBe(OperationRunOutcome::PartiallySucceeded->value)
|
|
->and($result?->reasonCode)->toBe('restore.verification_required');
|
|
});
|
|
|
|
it('fails closed when a completed restore lacks repo-backed result proof in Spec364', function (): void {
|
|
[, , $run] = spec364RestoreFixture([
|
|
'results' => [],
|
|
'metadata' => [],
|
|
]);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('failed_unrecoverable')
|
|
->and($result?->outcome)->toBe(OperationRunOutcome::Failed->value)
|
|
->and($result?->reasonCode)->toBe('restore.results_missing');
|
|
});
|
|
|
|
it('fails failed restore records with restore-specific reason metadata in Spec364', function (): void {
|
|
[, , $run] = spec364RestoreFixture([
|
|
'status' => RestoreRunStatus::Failed->value,
|
|
'failure_reason' => 'Provider rejected the restore.',
|
|
'results' => [
|
|
'foundations' => [],
|
|
'items' => [
|
|
['status' => 'failed', 'policy_identifier' => 'policy-failed'],
|
|
],
|
|
],
|
|
'metadata' => [
|
|
'total' => 1,
|
|
'succeeded' => 0,
|
|
'failed' => 1,
|
|
'skipped' => 0,
|
|
],
|
|
]);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('failed_unrecoverable')
|
|
->and($result?->outcome)->toBe(OperationRunOutcome::Failed->value)
|
|
->and($result?->reasonCode)->toBe('restore.failed')
|
|
->and($result?->reasonMessage)->toBe('Provider rejected the restore.');
|
|
});
|
|
|
|
it('refuses to reconcile restore runs outside the operation scope in Spec364', function (): void {
|
|
[, $tenant] = createUserWithTenant(role: 'owner');
|
|
$foreignTenant = ManagedEnvironment::factory()->create();
|
|
$foreignBackupSet = BackupSet::factory()->for($foreignTenant)->create();
|
|
|
|
$foreignRestoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()
|
|
->for($foreignTenant, 'tenant')
|
|
->for($foreignBackupSet)
|
|
->create([
|
|
'workspace_id' => (int) $foreignTenant->workspace_id,
|
|
]));
|
|
|
|
$run = OperationRun::factory()->forTenant($tenant)->create([
|
|
'type' => 'restore.execute',
|
|
'status' => OperationRunStatus::Queued->value,
|
|
'outcome' => OperationRunOutcome::Pending->value,
|
|
'context' => [
|
|
'restore_run_id' => (int) $foreignRestoreRun->getKey(),
|
|
'backup_set_id' => (int) $foreignBackupSet->getKey(),
|
|
],
|
|
]);
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run);
|
|
|
|
expect($result?->decision)->toBe('not_reconciled')
|
|
->and($result?->reasonCode)->toBe('restore.scope_mismatch')
|
|
->and(OperationRunLinks::related($run, $tenant))->not->toHaveKey('Restore Run')
|
|
->and(OperationRunResource::restoreContinuation($run))->toBeNull();
|
|
});
|
|
|
|
it('refuses to reconcile archived restore runs in Spec364', function (): void {
|
|
[, , $run, $restoreRun] = spec364RestoreFixture();
|
|
|
|
$restoreRun->delete();
|
|
|
|
$result = app(RestoreExecuteReconciliationAdapter::class)->reconcile($run->fresh());
|
|
|
|
expect($result?->decision)->toBe('not_reconciled')
|
|
->and($result?->reasonCode)->toBe('restore.run_deleted');
|
|
});
|
|
|
|
it('applies restore verification gaps through the service-owned reconciliation write path in Spec364', function (): void {
|
|
[, $tenant, $run, $restoreRun] = spec364RestoreFixture(runOverrides: [
|
|
'created_at' => now()->subMinutes(20),
|
|
]);
|
|
|
|
spec364RestoreAuditFor($tenant, $run, $restoreRun);
|
|
|
|
$change = app(AdapterRunReconciler::class)->reconcileOperationRun($run, false);
|
|
|
|
expect($change['applied'] ?? null)->toBeTrue()
|
|
->and($change['reason_code'] ?? null)->toBe('restore.verification_required');
|
|
|
|
$run->refresh();
|
|
|
|
expect($run->status)->toBe(OperationRunStatus::Completed->value)
|
|
->and($run->outcome)->toBe(OperationRunOutcome::PartiallySucceeded->value)
|
|
->and($run->reconciliationAdapter())->toBe('restore_run')
|
|
->and($run->reconciliationDecision())->toBe('reconciled_partially_succeeded')
|
|
->and($run->lifecycleReconciliationReasonCode())->toBe('restore.verification_required')
|
|
->and(data_get($run->failure_summary, '0.reason_code'))->toBe('restore.verification_required')
|
|
->and($run->summary_counts)->toMatchArray([
|
|
'total' => 1,
|
|
'processed' => 1,
|
|
'succeeded' => 1,
|
|
'failed' => 0,
|
|
'skipped' => 0,
|
|
])
|
|
->and($run->summary_counts)->not->toHaveKeys(['partial', 'non_applied']);
|
|
});
|
|
|
|
it('keeps high-risk unsupported operation families out of restore reconciliation in Spec364', function (): void {
|
|
expect(app(AdapterRunReconciler::class)->supportedTypes())
|
|
->toContain('restore.execute')
|
|
->not->toContain('restore.verify', 'restore.rollback.execute', 'promotion.execute', 'ai.execution');
|
|
});
|
|
|
|
/**
|
|
* @return array{0:mixed,1:ManagedEnvironment,2:OperationRun,3:RestoreRun,4:BackupSet}
|
|
*/
|
|
function spec364RestoreFixture(array $restoreOverrides = [], array $runOverrides = []): array
|
|
{
|
|
[$user, $tenant] = createUserWithTenant(role: 'owner');
|
|
|
|
$backupSet = BackupSet::factory()->for($tenant)->create([
|
|
'status' => 'completed',
|
|
]);
|
|
|
|
$run = OperationRun::factory()->forTenant($tenant)->create(array_replace_recursive([
|
|
'type' => 'restore.execute',
|
|
'status' => OperationRunStatus::Queued->value,
|
|
'outcome' => OperationRunOutcome::Pending->value,
|
|
'context' => [],
|
|
'started_at' => null,
|
|
'completed_at' => null,
|
|
], $runOverrides));
|
|
|
|
$restoreRun = RestoreRun::withoutEvents(fn (): RestoreRun => RestoreRun::factory()
|
|
->for($tenant, 'tenant')
|
|
->for($backupSet)
|
|
->create(array_replace([
|
|
'workspace_id' => (int) $tenant->workspace_id,
|
|
'operation_run_id' => (int) $run->getKey(),
|
|
'status' => RestoreRunStatus::Completed->value,
|
|
'is_dry_run' => false,
|
|
'started_at' => now()->subMinutes(5),
|
|
'completed_at' => now()->subMinute(),
|
|
'results' => [
|
|
'foundations' => [],
|
|
'items' => [
|
|
['status' => 'applied', 'policy_identifier' => 'policy-applied'],
|
|
],
|
|
],
|
|
'metadata' => [
|
|
'total' => 1,
|
|
'succeeded' => 1,
|
|
'failed' => 0,
|
|
'skipped' => 0,
|
|
'partial' => 0,
|
|
'non_applied' => 0,
|
|
],
|
|
], $restoreOverrides)));
|
|
|
|
$context = is_array($run->context) ? $run->context : [];
|
|
$run->forceFill([
|
|
'context' => array_replace($context, [
|
|
'restore_run_id' => (int) $restoreRun->getKey(),
|
|
'backup_set_id' => (int) $backupSet->getKey(),
|
|
'is_dry_run' => (bool) ($restoreRun->is_dry_run ?? false),
|
|
]),
|
|
])->save();
|
|
|
|
return [$user, $tenant, $run->fresh(), $restoreRun->fresh(), $backupSet];
|
|
}
|
|
|
|
function spec364EvidenceSnapshotFor(ManagedEnvironment $tenant, OperationRun $run): EvidenceSnapshot
|
|
{
|
|
return EvidenceSnapshot::query()->create([
|
|
'workspace_id' => (int) $tenant->workspace_id,
|
|
'managed_environment_id' => (int) $tenant->getKey(),
|
|
'operation_run_id' => (int) $run->getKey(),
|
|
'fingerprint' => hash('sha256', 'spec364-evidence-'.$run->getKey()),
|
|
'status' => EvidenceSnapshotStatus::Active->value,
|
|
'completeness_state' => EvidenceCompletenessState::Complete->value,
|
|
'summary' => [
|
|
'source' => 'spec364',
|
|
'restore_run_id' => (int) data_get($run->context, 'restore_run_id'),
|
|
],
|
|
'generated_at' => now(),
|
|
]);
|
|
}
|
|
|
|
function spec364RestoreAuditFor(ManagedEnvironment $tenant, OperationRun $run, RestoreRun $restoreRun): AuditLog
|
|
{
|
|
return AuditLog::query()->create([
|
|
'workspace_id' => (int) $tenant->workspace_id,
|
|
'managed_environment_id' => (int) $tenant->getKey(),
|
|
'operation_run_id' => (int) $run->getKey(),
|
|
'actor_id' => null,
|
|
'actor_email' => null,
|
|
'actor_name' => null,
|
|
'action' => 'restore.executed',
|
|
'resource_type' => 'restore_run',
|
|
'resource_id' => (string) $restoreRun->getKey(),
|
|
'status' => 'success',
|
|
'metadata' => [
|
|
'restore_run_id' => (int) $restoreRun->getKey(),
|
|
'backup_set_id' => (int) $restoreRun->backup_set_id,
|
|
'status' => (string) $restoreRun->status,
|
|
],
|
|
'recorded_at' => now(),
|
|
]);
|
|
}
|