From 97e06587bfc6c15266c2a00f83321c725380f14f Mon Sep 17 00:00:00 2001 From: Ahmed Darrazi Date: Wed, 4 Feb 2026 00:57:26 +0100 Subject: [PATCH] feat: verification report framework --- .../ManagedTenantOnboardingWizard.php | 88 ++++- .../Resources/OperationRunResource.php | 26 +- .../Resources/ProviderConnectionResource.php | 25 +- .../Pages/EditProviderConnection.php | 15 +- .../Support/VerificationReportViewer.php | 44 +++ app/Jobs/ProviderConnectionHealthCheckJob.php | 86 ++++- .../Verification/StartVerification.php | 57 +++ app/Support/Audit/AuditActionId.php | 2 + app/Support/Badges/BadgeCatalog.php | 3 + app/Support/Badges/BadgeDomain.php | 3 + .../VerificationCheckSeverityBadge.php | 25 ++ .../Domains/VerificationCheckStatusBadge.php | 25 ++ .../VerificationReportOverallBadge.php | 24 ++ .../VerificationCheckSeverity.php | 20 + .../Verification/VerificationCheckStatus.php | 20 + .../VerificationReportOverall.php | 19 + .../VerificationReportSanitizer.php | 358 ++++++++++++++++++ .../Verification/VerificationReportSchema.php | 235 ++++++++++++ .../Verification/VerificationReportWriter.php | 343 +++++++++++++++++ .../verification-report-viewer.blade.php | 178 +++++++++ .../checklists/requirements.md | 34 ++ .../contracts/examples/fail.json | 47 +++ .../contracts/examples/pass.json | 29 ++ .../contracts/examples/running.json | 51 +++ .../contracts/examples/warn.json | 42 ++ .../contracts/reason-codes.md | 26 ++ .../contracts/verification-report.schema.json | 128 +++++++ .../074-verification-checklist/data-model.md | 61 +++ specs/074-verification-checklist/plan.md | 127 +++++++ .../074-verification-checklist/quickstart.md | 79 ++++ specs/074-verification-checklist/research.md | 86 +++++ specs/074-verification-checklist/spec.md | 186 +++++++++ specs/074-verification-checklist/tasks.md | 120 ++++++ ...rConnectionHealthCheckWritesReportTest.php | 165 ++++++++ .../VerificationAuthorizationTest.php | 112 ++++++ ...rificationReportMissingOrMalformedTest.php | 59 +++ .../VerificationReportRedactionTest.php | 47 +++ .../VerificationReportViewerDbOnlyTest.php | 50 +++ .../VerificationStartAfterCompletionTest.php | 62 +++ .../VerificationStartDedupeTest.php | 53 +++ tests/Unit/Badges/VerificationBadgesTest.php | 68 ++++ 41 files changed, 3178 insertions(+), 50 deletions(-) create mode 100644 app/Filament/Support/VerificationReportViewer.php create mode 100644 app/Services/Verification/StartVerification.php create mode 100644 app/Support/Badges/Domains/VerificationCheckSeverityBadge.php create mode 100644 app/Support/Badges/Domains/VerificationCheckStatusBadge.php create mode 100644 app/Support/Badges/Domains/VerificationReportOverallBadge.php create mode 100644 app/Support/Verification/VerificationCheckSeverity.php create mode 100644 app/Support/Verification/VerificationCheckStatus.php create mode 100644 app/Support/Verification/VerificationReportOverall.php create mode 100644 app/Support/Verification/VerificationReportSanitizer.php create mode 100644 app/Support/Verification/VerificationReportSchema.php create mode 100644 app/Support/Verification/VerificationReportWriter.php create mode 100644 resources/views/filament/components/verification-report-viewer.blade.php create mode 100644 specs/074-verification-checklist/checklists/requirements.md create mode 100644 specs/074-verification-checklist/contracts/examples/fail.json create mode 100644 specs/074-verification-checklist/contracts/examples/pass.json create mode 100644 specs/074-verification-checklist/contracts/examples/running.json create mode 100644 specs/074-verification-checklist/contracts/examples/warn.json create mode 100644 specs/074-verification-checklist/contracts/reason-codes.md create mode 100644 specs/074-verification-checklist/contracts/verification-report.schema.json create mode 100644 specs/074-verification-checklist/data-model.md create mode 100644 specs/074-verification-checklist/plan.md create mode 100644 specs/074-verification-checklist/quickstart.md create mode 100644 specs/074-verification-checklist/research.md create mode 100644 specs/074-verification-checklist/spec.md create mode 100644 specs/074-verification-checklist/tasks.md create mode 100644 tests/Feature/Verification/ProviderConnectionHealthCheckWritesReportTest.php create mode 100644 tests/Feature/Verification/VerificationAuthorizationTest.php create mode 100644 tests/Feature/Verification/VerificationReportMissingOrMalformedTest.php create mode 100644 tests/Feature/Verification/VerificationReportRedactionTest.php create mode 100644 tests/Feature/Verification/VerificationReportViewerDbOnlyTest.php create mode 100644 tests/Feature/Verification/VerificationStartAfterCompletionTest.php create mode 100644 tests/Feature/Verification/VerificationStartDedupeTest.php create mode 100644 tests/Unit/Badges/VerificationBadgesTest.php diff --git a/app/Filament/Pages/Workspaces/ManagedTenantOnboardingWizard.php b/app/Filament/Pages/Workspaces/ManagedTenantOnboardingWizard.php index a8b9b7e..bde5ff5 100644 --- a/app/Filament/Pages/Workspaces/ManagedTenantOnboardingWizard.php +++ b/app/Filament/Pages/Workspaces/ManagedTenantOnboardingWizard.php @@ -5,8 +5,8 @@ namespace App\Filament\Pages\Workspaces; use App\Filament\Pages\TenantDashboard; +use App\Filament\Support\VerificationReportViewer; use App\Jobs\ProviderComplianceSnapshotJob; -use App\Jobs\ProviderConnectionHealthCheckJob; use App\Jobs\ProviderInventorySyncJob; use App\Models\OperationRun; use App\Models\ProviderConnection; @@ -15,15 +15,17 @@ use App\Models\TenantOnboardingSession; use App\Models\User; use App\Models\Workspace; +use App\Models\WorkspaceMembership; use App\Services\Audit\WorkspaceAuditLogger; use App\Services\Auth\TenantMembershipManager; use App\Services\OperationRunService; use App\Services\Providers\CredentialManager; use App\Services\Providers\ProviderOperationRegistry; -use App\Services\Providers\ProviderOperationStartGate; +use App\Services\Verification\StartVerification; use App\Support\Audit\AuditActionId; use App\Support\Auth\Capabilities; use App\Support\OperationRunLinks; +use App\Support\Rbac\UiEnforcement; use App\Support\Workspaces\WorkspaceContext; use Filament\Actions\Action; use Filament\Forms\Components\CheckboxList; @@ -37,6 +39,7 @@ use Filament\Schemas\Components\Section; use Filament\Schemas\Components\Text; use Filament\Schemas\Components\Utilities\Get; +use Filament\Schemas\Components\View; use Filament\Schemas\Components\Wizard; use Filament\Schemas\Components\Wizard\Step; use Filament\Schemas\Schema; @@ -236,14 +239,21 @@ public function content(Schema $schema): Schema ->schema([ Section::make('Verification') ->schema([ - Text::make(fn (): string => 'Status: '.$this->verificationStatusLabel()) - ->badge() - ->color(fn (): string => $this->verificationHasSucceeded() ? 'success' : 'warning'), + View::make('filament.components.verification-report-viewer') + ->viewData(fn (): array => [ + 'report' => $this->verificationReport(), + ]), SchemaActions::make([ - Action::make('wizardStartVerification') - ->label('Start verification') - ->visible(fn (): bool => $this->managedTenant instanceof Tenant) - ->action(fn () => $this->startVerification()), + UiEnforcement::forTableAction( + Action::make('wizardStartVerification') + ->label('Start verification') + ->visible(fn (): bool => $this->managedTenant instanceof Tenant) + ->action(fn () => $this->startVerification()), + fn (): ?Tenant => $this->managedTenant, + ) + ->preserveVisibility() + ->requireCapability(Capabilities::PROVIDER_RUN) + ->apply(), Action::make('wizardViewVerificationRun') ->label('View run') ->url(fn (): ?string => $this->verificationRunUrl()) @@ -467,6 +477,37 @@ private function verificationRunUrl(): ?string return OperationRunLinks::view($runId, $this->managedTenant); } + /** + * @return array|null + */ + private function verificationReport(): ?array + { + if (! $this->managedTenant instanceof Tenant) { + return null; + } + + if (! $this->onboardingSession instanceof TenantOnboardingSession) { + return null; + } + + $runId = $this->onboardingSession->state['verification_operation_run_id'] ?? null; + + if (! is_int($runId)) { + return null; + } + + $run = OperationRun::query() + ->where('tenant_id', (int) $this->managedTenant->getKey()) + ->whereKey($runId) + ->first(); + + if (! $run instanceof OperationRun) { + return null; + } + + return VerificationReportViewer::report($run); + } + private function bootstrapRunsLabel(): string { if (! $this->onboardingSession instanceof TenantOnboardingSession) { @@ -819,6 +860,24 @@ public function startVerification(): void abort(404); } + if (! $user->canAccessTenant($tenant)) { + $workspaceMembership = WorkspaceMembership::query() + ->where('workspace_id', (int) $this->workspace->getKey()) + ->where('user_id', (int) $user->getKey()) + ->first(); + + $role = is_string($workspaceMembership?->role ?? null) ? (string) $workspaceMembership->role : 'readonly'; + + app(TenantMembershipManager::class)->addMember( + tenant: $tenant, + actor: $user, + member: $user, + role: $role, + source: 'manual', + sourceRef: 'managed_tenant_onboarding', + ); + } + $connection = $this->resolveSelectedProviderConnection($tenant); if (! $connection instanceof ProviderConnection) { @@ -831,18 +890,9 @@ public function startVerification(): void return; } - $result = app(ProviderOperationStartGate::class)->start( + $result = app(StartVerification::class)->providerConnectionCheck( tenant: $tenant, connection: $connection, - operationType: 'provider.connection.check', - dispatcher: function (OperationRun $run) use ($tenant, $user, $connection): void { - ProviderConnectionHealthCheckJob::dispatch( - tenantId: (int) $tenant->getKey(), - userId: (int) $user->getKey(), - providerConnectionId: (int) $connection->getKey(), - operationRun: $run, - ); - }, initiator: $user, extraContext: [ 'wizard' => [ diff --git a/app/Filament/Resources/OperationRunResource.php b/app/Filament/Resources/OperationRunResource.php index 012b33c..1168a13 100644 --- a/app/Filament/Resources/OperationRunResource.php +++ b/app/Filament/Resources/OperationRunResource.php @@ -3,6 +3,7 @@ namespace App\Filament\Resources; use App\Filament\Resources\OperationRunResource\Pages; +use App\Filament\Support\VerificationReportViewer; use App\Models\OperationRun; use App\Models\Tenant; use App\Support\Badges\BadgeDomain; @@ -136,12 +137,35 @@ public static function infolist(Schema $schema): Schema ->visible(fn (OperationRun $record): bool => ! empty($record->failure_summary)) ->columnSpanFull(), + Section::make('Verification report') + ->schema([ + ViewEntry::make('verification_report') + ->label('') + ->view('filament.components.verification-report-viewer') + ->state(fn (OperationRun $record): ?array => VerificationReportViewer::report($record)) + ->columnSpanFull(), + ]) + ->visible(fn (OperationRun $record): bool => VerificationReportViewer::shouldRenderForRun($record)) + ->columnSpanFull(), + Section::make('Context') ->schema([ ViewEntry::make('context') ->label('') ->view('filament.infolists.entries.snapshot-json') - ->state(fn (OperationRun $record): array => $record->context ?? []) + ->state(function (OperationRun $record): array { + $context = $record->context ?? []; + $context = is_array($context) ? $context : []; + + if (array_key_exists('verification_report', $context)) { + $context['verification_report'] = [ + 'redacted' => true, + 'note' => 'Rendered in the Verification report section.', + ]; + } + + return $context; + }) ->columnSpanFull(), ]) ->columnSpanFull(), diff --git a/app/Filament/Resources/ProviderConnectionResource.php b/app/Filament/Resources/ProviderConnectionResource.php index 7153fa9..bd31cca 100644 --- a/app/Filament/Resources/ProviderConnectionResource.php +++ b/app/Filament/Resources/ProviderConnectionResource.php @@ -5,7 +5,6 @@ use App\Filament\Concerns\ScopesGlobalSearchToTenant; use App\Filament\Resources\ProviderConnectionResource\Pages; use App\Jobs\ProviderComplianceSnapshotJob; -use App\Jobs\ProviderConnectionHealthCheckJob; use App\Jobs\ProviderInventorySyncJob; use App\Models\OperationRun; use App\Models\ProviderConnection; @@ -15,6 +14,7 @@ use App\Services\Intune\AuditLogger; use App\Services\Providers\CredentialManager; use App\Services\Providers\ProviderOperationStartGate; +use App\Services\Verification\StartVerification; use App\Support\Auth\Capabilities; use App\Support\Badges\BadgeDomain; use App\Support\Badges\BadgeRenderer; @@ -175,29 +175,22 @@ public static function table(Table $table): Table ->icon('heroicon-o-check-badge') ->color('success') ->visible(fn (ProviderConnection $record): bool => $record->status !== 'disabled') - ->action(function (ProviderConnection $record, ProviderOperationStartGate $gate): void { + ->action(function (ProviderConnection $record, StartVerification $verification): void { $tenant = Tenant::current(); $user = auth()->user(); - if (! $tenant instanceof Tenant || ! $user instanceof User) { - return; + if (! $tenant instanceof Tenant) { + abort(404); } - $initiator = $user; + if (! $user instanceof User) { + abort(403); + } - $result = $gate->start( + $result = $verification->providerConnectionCheck( tenant: $tenant, connection: $record, - operationType: 'provider.connection.check', - dispatcher: function (OperationRun $operationRun) use ($tenant, $initiator, $record): void { - ProviderConnectionHealthCheckJob::dispatch( - tenantId: (int) $tenant->getKey(), - userId: (int) $initiator->getKey(), - providerConnectionId: (int) $record->getKey(), - operationRun: $operationRun, - ); - }, - initiator: $initiator, + initiator: $user, ); if ($result->status === 'scope_busy') { diff --git a/app/Filament/Resources/ProviderConnectionResource/Pages/EditProviderConnection.php b/app/Filament/Resources/ProviderConnectionResource/Pages/EditProviderConnection.php index 3b97d2c..6afef09 100644 --- a/app/Filament/Resources/ProviderConnectionResource/Pages/EditProviderConnection.php +++ b/app/Filament/Resources/ProviderConnectionResource/Pages/EditProviderConnection.php @@ -4,7 +4,6 @@ use App\Filament\Resources\ProviderConnectionResource; use App\Jobs\ProviderComplianceSnapshotJob; -use App\Jobs\ProviderConnectionHealthCheckJob; use App\Jobs\ProviderInventorySyncJob; use App\Models\OperationRun; use App\Models\ProviderConnection; @@ -14,6 +13,7 @@ use App\Services\Intune\AuditLogger; use App\Services\Providers\CredentialManager; use App\Services\Providers\ProviderOperationStartGate; +use App\Services\Verification\StartVerification; use App\Support\Auth\Capabilities; use App\Support\OperationRunLinks; use App\Support\Rbac\UiEnforcement; @@ -167,7 +167,7 @@ protected function getHeaderActions(): array && $user->canAccessTenant($tenant) && $record->status !== 'disabled'; }) - ->action(function (ProviderConnection $record, ProviderOperationStartGate $gate): void { + ->action(function (ProviderConnection $record, StartVerification $verification): void { $tenant = Tenant::current(); $user = auth()->user(); @@ -185,18 +185,9 @@ protected function getHeaderActions(): array $initiator = $user; - $result = $gate->start( + $result = $verification->providerConnectionCheck( tenant: $tenant, connection: $record, - operationType: 'provider.connection.check', - dispatcher: function (OperationRun $operationRun) use ($tenant, $initiator, $record): void { - ProviderConnectionHealthCheckJob::dispatch( - tenantId: (int) $tenant->getKey(), - userId: (int) $initiator->getKey(), - providerConnectionId: (int) $record->getKey(), - operationRun: $operationRun, - ); - }, initiator: $initiator, ); diff --git a/app/Filament/Support/VerificationReportViewer.php b/app/Filament/Support/VerificationReportViewer.php new file mode 100644 index 0000000..aaa766f --- /dev/null +++ b/app/Filament/Support/VerificationReportViewer.php @@ -0,0 +1,44 @@ +|null + */ + public static function report(OperationRun $run): ?array + { + $context = is_array($run->context) ? $run->context : []; + $report = $context['verification_report'] ?? null; + + if (! is_array($report)) { + return null; + } + + $report = VerificationReportSanitizer::sanitizeReport($report); + + if (! VerificationReportSchema::isValidReport($report)) { + return null; + } + + return $report; + } + + public static function shouldRenderForRun(OperationRun $run): bool + { + $context = is_array($run->context) ? $run->context : []; + + if (array_key_exists('verification_report', $context)) { + return true; + } + + return in_array((string) $run->type, ['provider.connection.check'], true); + } +} diff --git a/app/Jobs/ProviderConnectionHealthCheckJob.php b/app/Jobs/ProviderConnectionHealthCheckJob.php index 8afa684..0c61c22 100644 --- a/app/Jobs/ProviderConnectionHealthCheckJob.php +++ b/app/Jobs/ProviderConnectionHealthCheckJob.php @@ -7,11 +7,14 @@ use App\Models\ProviderConnection; use App\Models\Tenant; use App\Models\User; +use App\Services\Audit\WorkspaceAuditLogger; use App\Services\OperationRunService; use App\Services\Providers\Contracts\HealthResult; use App\Services\Providers\MicrosoftProviderHealthCheck; +use App\Support\Audit\AuditActionId; use App\Support\OperationRunOutcome; use App\Support\OperationRunStatus; +use App\Support\Verification\VerificationReportWriter; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; @@ -83,17 +86,64 @@ public function handle( $this->updateRunTargetScope($this->operationRun, $connection, $entraTenantName); + $report = VerificationReportWriter::write( + run: $this->operationRun, + checks: [ + [ + 'key' => 'provider.connection.check', + 'title' => 'Provider connection check', + 'status' => $result->healthy ? 'pass' : 'fail', + 'severity' => $result->healthy ? 'info' : 'critical', + 'blocking' => ! $result->healthy, + 'reason_code' => $result->healthy ? 'ok' : ($result->reasonCode ?? 'unknown_error'), + 'message' => $result->healthy ? 'Connection is healthy.' : ($result->message ?? 'Health check failed.'), + 'evidence' => array_values(array_filter([ + [ + 'kind' => 'provider_connection_id', + 'value' => (int) $connection->getKey(), + ], + [ + 'kind' => 'entra_tenant_id', + 'value' => (string) $connection->entra_tenant_id, + ], + is_numeric($result->meta['http_status'] ?? null) ? [ + 'kind' => 'http_status', + 'value' => (int) $result->meta['http_status'], + ] : null, + is_string($result->meta['organization_id'] ?? null) ? [ + 'kind' => 'organization_id', + 'value' => (string) $result->meta['organization_id'], + ] : null, + ])), + 'next_steps' => $result->healthy + ? [] + : [[ + 'label' => 'Review provider connection', + 'url' => \App\Filament\Resources\ProviderConnectionResource::getUrl('edit', [ + 'record' => (int) $connection->getKey(), + ], tenant: $tenant), + ]], + ], + ], + identity: [ + 'provider_connection_id' => (int) $connection->getKey(), + 'entra_tenant_id' => (string) $connection->entra_tenant_id, + ], + ); + if ($result->healthy) { - $runs->updateRun( + $run = $runs->updateRun( $this->operationRun, status: OperationRunStatus::Completed->value, outcome: OperationRunOutcome::Succeeded->value, ); + $this->logVerificationCompletion($tenant, $user, $run, $report); + return; } - $runs->updateRun( + $run = $runs->updateRun( $this->operationRun, status: OperationRunStatus::Completed->value, outcome: OperationRunOutcome::Failed->value, @@ -103,6 +153,8 @@ public function handle( 'message' => $result->message ?? 'Health check failed.', ]], ); + + $this->logVerificationCompletion($tenant, $user, $run, $report); } private function resolveEntraTenantName(ProviderConnection $connection, HealthResult $result): ?string @@ -145,4 +197,34 @@ private function applyHealthResult(ProviderConnection $connection, HealthResult 'last_error_message' => $result->healthy ? null : $result->message, ]); } + + /** + * @param array $report + */ + private function logVerificationCompletion(Tenant $tenant, User $actor, OperationRun $run, array $report): void + { + $workspace = $tenant->workspace; + + if (! $workspace) { + return; + } + + $counts = $report['summary']['counts'] ?? []; + $counts = is_array($counts) ? $counts : []; + + app(WorkspaceAuditLogger::class)->log( + workspace: $workspace, + action: AuditActionId::VerificationCompleted->value, + context: [ + 'metadata' => [ + 'operation_run_id' => (int) $run->getKey(), + 'counts' => $counts, + ], + ], + actor: $actor, + status: $run->outcome === OperationRunOutcome::Succeeded->value ? 'success' : 'failed', + resourceType: 'operation_run', + resourceId: (string) $run->getKey(), + ); + } } diff --git a/app/Services/Verification/StartVerification.php b/app/Services/Verification/StartVerification.php new file mode 100644 index 0000000..2290fd1 --- /dev/null +++ b/app/Services/Verification/StartVerification.php @@ -0,0 +1,57 @@ + $extraContext + */ + public function providerConnectionCheck( + Tenant $tenant, + ProviderConnection $connection, + User $initiator, + array $extraContext = [], + ): ProviderOperationStartResult { + if (! $initiator->canAccessTenant($tenant)) { + throw new NotFoundHttpException; + } + + Gate::forUser($initiator)->authorize(Capabilities::PROVIDER_RUN, $tenant); + + return $this->providers->start( + tenant: $tenant, + connection: $connection, + operationType: 'provider.connection.check', + dispatcher: function (OperationRun $run) use ($tenant, $initiator, $connection): void { + ProviderConnectionHealthCheckJob::dispatch( + tenantId: (int) $tenant->getKey(), + userId: (int) $initiator->getKey(), + providerConnectionId: (int) $connection->getKey(), + operationRun: $run, + ); + }, + initiator: $initiator, + extraContext: $extraContext, + ); + } +} diff --git a/app/Support/Audit/AuditActionId.php b/app/Support/Audit/AuditActionId.php index 16c1302..b63a551 100644 --- a/app/Support/Audit/AuditActionId.php +++ b/app/Support/Audit/AuditActionId.php @@ -27,4 +27,6 @@ enum AuditActionId: string case ManagedTenantOnboardingStart = 'managed_tenant_onboarding.start'; case ManagedTenantOnboardingResume = 'managed_tenant_onboarding.resume'; case ManagedTenantOnboardingVerificationStart = 'managed_tenant_onboarding.verification_start'; + + case VerificationCompleted = 'verification.completed'; } diff --git a/app/Support/Badges/BadgeCatalog.php b/app/Support/Badges/BadgeCatalog.php index 0432584..47e6b76 100644 --- a/app/Support/Badges/BadgeCatalog.php +++ b/app/Support/Badges/BadgeCatalog.php @@ -36,6 +36,9 @@ final class BadgeCatalog BadgeDomain::RestoreResultStatus->value => Domains\RestoreResultStatusBadge::class, BadgeDomain::ProviderConnectionStatus->value => Domains\ProviderConnectionStatusBadge::class, BadgeDomain::ProviderConnectionHealth->value => Domains\ProviderConnectionHealthBadge::class, + BadgeDomain::VerificationCheckStatus->value => Domains\VerificationCheckStatusBadge::class, + BadgeDomain::VerificationCheckSeverity->value => Domains\VerificationCheckSeverityBadge::class, + BadgeDomain::VerificationReportOverall->value => Domains\VerificationReportOverallBadge::class, ]; /** diff --git a/app/Support/Badges/BadgeDomain.php b/app/Support/Badges/BadgeDomain.php index 8e0710a..89a41a9 100644 --- a/app/Support/Badges/BadgeDomain.php +++ b/app/Support/Badges/BadgeDomain.php @@ -28,4 +28,7 @@ enum BadgeDomain: string case RestoreResultStatus = 'restore_result_status'; case ProviderConnectionStatus = 'provider_connection.status'; case ProviderConnectionHealth = 'provider_connection.health'; + case VerificationCheckStatus = 'verification_check_status'; + case VerificationCheckSeverity = 'verification_check_severity'; + case VerificationReportOverall = 'verification_report_overall'; } diff --git a/app/Support/Badges/Domains/VerificationCheckSeverityBadge.php b/app/Support/Badges/Domains/VerificationCheckSeverityBadge.php new file mode 100644 index 0000000..f15bd08 --- /dev/null +++ b/app/Support/Badges/Domains/VerificationCheckSeverityBadge.php @@ -0,0 +1,25 @@ +value => new BadgeSpec('Info', 'gray', 'heroicon-m-information-circle'), + VerificationCheckSeverity::Low->value => new BadgeSpec('Low', 'info', 'heroicon-m-arrow-down'), + VerificationCheckSeverity::Medium->value => new BadgeSpec('Medium', 'warning', 'heroicon-m-exclamation-triangle'), + VerificationCheckSeverity::High->value => new BadgeSpec('High', 'danger', 'heroicon-m-exclamation-triangle'), + VerificationCheckSeverity::Critical->value => new BadgeSpec('Critical', 'danger', 'heroicon-m-x-circle'), + default => BadgeSpec::unknown(), + }; + } +} diff --git a/app/Support/Badges/Domains/VerificationCheckStatusBadge.php b/app/Support/Badges/Domains/VerificationCheckStatusBadge.php new file mode 100644 index 0000000..f8be3ef --- /dev/null +++ b/app/Support/Badges/Domains/VerificationCheckStatusBadge.php @@ -0,0 +1,25 @@ +value => new BadgeSpec('Pass', 'success', 'heroicon-m-check-circle'), + VerificationCheckStatus::Fail->value => new BadgeSpec('Fail', 'danger', 'heroicon-m-x-circle'), + VerificationCheckStatus::Warn->value => new BadgeSpec('Warn', 'warning', 'heroicon-m-exclamation-triangle'), + VerificationCheckStatus::Skip->value => new BadgeSpec('Skipped', 'gray', 'heroicon-m-minus-circle'), + VerificationCheckStatus::Running->value => new BadgeSpec('Running', 'info', 'heroicon-m-arrow-path'), + default => BadgeSpec::unknown(), + }; + } +} diff --git a/app/Support/Badges/Domains/VerificationReportOverallBadge.php b/app/Support/Badges/Domains/VerificationReportOverallBadge.php new file mode 100644 index 0000000..87f60b4 --- /dev/null +++ b/app/Support/Badges/Domains/VerificationReportOverallBadge.php @@ -0,0 +1,24 @@ +value => new BadgeSpec('Ready', 'success', 'heroicon-m-check-circle'), + VerificationReportOverall::NeedsAttention->value => new BadgeSpec('Needs attention', 'warning', 'heroicon-m-exclamation-triangle'), + VerificationReportOverall::Blocked->value => new BadgeSpec('Blocked', 'danger', 'heroicon-m-x-circle'), + VerificationReportOverall::Running->value => new BadgeSpec('Running', 'info', 'heroicon-m-arrow-path'), + default => BadgeSpec::unknown(), + }; + } +} diff --git a/app/Support/Verification/VerificationCheckSeverity.php b/app/Support/Verification/VerificationCheckSeverity.php new file mode 100644 index 0000000..627ac44 --- /dev/null +++ b/app/Support/Verification/VerificationCheckSeverity.php @@ -0,0 +1,20 @@ + + */ + public static function values(): array + { + return array_map(static fn (self $case): string => $case->value, self::cases()); + } +} diff --git a/app/Support/Verification/VerificationCheckStatus.php b/app/Support/Verification/VerificationCheckStatus.php new file mode 100644 index 0000000..148a9d7 --- /dev/null +++ b/app/Support/Verification/VerificationCheckStatus.php @@ -0,0 +1,20 @@ + + */ + public static function values(): array + { + return array_map(static fn (self $case): string => $case->value, self::cases()); + } +} diff --git a/app/Support/Verification/VerificationReportOverall.php b/app/Support/Verification/VerificationReportOverall.php new file mode 100644 index 0000000..982faab --- /dev/null +++ b/app/Support/Verification/VerificationReportOverall.php @@ -0,0 +1,19 @@ + + */ + public static function values(): array + { + return array_map(static fn (self $case): string => $case->value, self::cases()); + } +} diff --git a/app/Support/Verification/VerificationReportSanitizer.php b/app/Support/Verification/VerificationReportSanitizer.php new file mode 100644 index 0000000..ad9ddda --- /dev/null +++ b/app/Support/Verification/VerificationReportSanitizer.php @@ -0,0 +1,358 @@ + + */ + private const FORBIDDEN_KEY_SUBSTRINGS = [ + 'access_token', + 'refresh_token', + 'client_secret', + 'authorization', + 'password', + 'cookie', + 'set-cookie', + ]; + + /** + * @return array + */ + public static function sanitizeReport(array $report): array + { + $sanitized = []; + + $schemaVersion = self::sanitizeShortString($report['schema_version'] ?? null, fallback: null); + if ($schemaVersion !== null) { + $sanitized['schema_version'] = $schemaVersion; + } + + $flow = self::sanitizeShortString($report['flow'] ?? null, fallback: null); + if ($flow !== null) { + $sanitized['flow'] = $flow; + } + + $generatedAt = self::sanitizeShortString($report['generated_at'] ?? null, fallback: null); + if ($generatedAt !== null) { + $sanitized['generated_at'] = $generatedAt; + } + + if (is_array($report['identity'] ?? null)) { + $identity = self::sanitizeIdentity((array) $report['identity']); + + if ($identity !== []) { + $sanitized['identity'] = $identity; + } + } + + $summary = is_array($report['summary'] ?? null) ? (array) $report['summary'] : []; + $summary = self::sanitizeSummary($summary); + + if ($summary !== null) { + $sanitized['summary'] = $summary; + } + + $checks = is_array($report['checks'] ?? null) ? (array) $report['checks'] : []; + $checks = self::sanitizeChecks($checks); + + if ($checks !== null) { + $sanitized['checks'] = $checks; + } + + return $sanitized; + } + + /** + * @param array $identity + * @return array + */ + private static function sanitizeIdentity(array $identity): array + { + $sanitized = []; + + foreach ($identity as $key => $value) { + if (! is_string($key) || trim($key) === '') { + continue; + } + + if (self::containsForbiddenKeySubstring($key)) { + continue; + } + + if (is_int($value)) { + $sanitized[$key] = $value; + + continue; + } + + if (! is_string($value)) { + continue; + } + + $value = self::sanitizeValueString($value); + + if ($value !== null) { + $sanitized[$key] = $value; + } + } + + return $sanitized; + } + + /** + * @param array $summary + * @return array{overall: string, counts: array{total: int, pass: int, fail: int, warn: int, skip: int, running: int}}|null + */ + private static function sanitizeSummary(array $summary): ?array + { + $overall = $summary['overall'] ?? null; + + if (! is_string($overall) || ! in_array($overall, VerificationReportOverall::values(), true)) { + return null; + } + + $counts = is_array($summary['counts'] ?? null) ? (array) $summary['counts'] : []; + + foreach (['total', 'pass', 'fail', 'warn', 'skip', 'running'] as $key) { + if (! is_int($counts[$key] ?? null) || $counts[$key] < 0) { + return null; + } + } + + return [ + 'overall' => $overall, + 'counts' => [ + 'total' => $counts['total'], + 'pass' => $counts['pass'], + 'fail' => $counts['fail'], + 'warn' => $counts['warn'], + 'skip' => $counts['skip'], + 'running' => $counts['running'], + ], + ]; + } + + /** + * @param array $checks + * @return array>|null + */ + private static function sanitizeChecks(array $checks): ?array + { + if ($checks === []) { + return []; + } + + $sanitized = []; + + foreach ($checks as $check) { + if (! is_array($check)) { + continue; + } + + $key = self::sanitizeShortString($check['key'] ?? null, fallback: null); + $title = self::sanitizeShortString($check['title'] ?? null, fallback: null); + $reasonCode = self::sanitizeShortString($check['reason_code'] ?? null, fallback: null); + + if ($key === null || $title === null || $reasonCode === null) { + continue; + } + + $status = $check['status'] ?? null; + if (! is_string($status) || ! in_array($status, VerificationCheckStatus::values(), true)) { + continue; + } + + $severity = $check['severity'] ?? null; + if (! is_string($severity) || ! in_array($severity, VerificationCheckSeverity::values(), true)) { + continue; + } + + $messageRaw = $check['message'] ?? null; + if (! is_string($messageRaw) || trim($messageRaw) === '') { + continue; + } + + $blocking = is_bool($check['blocking'] ?? null) ? (bool) $check['blocking'] : false; + + $sanitized[] = [ + 'key' => $key, + 'title' => $title, + 'status' => $status, + 'severity' => $severity, + 'blocking' => $blocking, + 'reason_code' => $reasonCode, + 'message' => self::sanitizeMessage($messageRaw), + 'evidence' => self::sanitizeEvidence(is_array($check['evidence'] ?? null) ? (array) $check['evidence'] : []), + 'next_steps' => self::sanitizeNextSteps(is_array($check['next_steps'] ?? null) ? (array) $check['next_steps'] : []), + ]; + } + + return $sanitized; + } + + /** + * @param array $evidence + * @return array + */ + private static function sanitizeEvidence(array $evidence): array + { + $sanitized = []; + + foreach ($evidence as $pointer) { + if (! is_array($pointer)) { + continue; + } + + $kind = $pointer['kind'] ?? null; + if (! is_string($kind) || trim($kind) === '') { + continue; + } + + if (self::containsForbiddenKeySubstring($kind)) { + continue; + } + + $value = $pointer['value'] ?? null; + + if (is_int($value)) { + $sanitized[] = ['kind' => trim($kind), 'value' => $value]; + + continue; + } + + if (! is_string($value)) { + continue; + } + + $sanitizedValue = self::sanitizeValueString($value); + + if ($sanitizedValue === null) { + continue; + } + + $sanitized[] = ['kind' => trim($kind), 'value' => $sanitizedValue]; + } + + return $sanitized; + } + + /** + * @param array $nextSteps + * @return array + */ + private static function sanitizeNextSteps(array $nextSteps): array + { + $sanitized = []; + + foreach ($nextSteps as $step) { + if (! is_array($step)) { + continue; + } + + $label = self::sanitizeShortString($step['label'] ?? null, fallback: null); + $url = self::sanitizeShortString($step['url'] ?? null, fallback: null); + + if ($label === null || $url === null) { + continue; + } + + $sanitized[] = [ + 'label' => $label, + 'url' => $url, + ]; + } + + return $sanitized; + } + + private static function sanitizeMessage(mixed $message): string + { + if (! is_string($message)) { + return '—'; + } + + $message = trim(str_replace(["\r", "\n"], ' ', $message)); + + $message = preg_replace('/\bAuthorization\s*:\s*[^\s]+(?:\s+[^\s]+)?/i', '[REDACTED_AUTH]', $message) ?? $message; + $message = preg_replace('/\bBearer\s+[A-Za-z0-9\-\._~\+\/]+=*\b/i', '[REDACTED_AUTH]', $message) ?? $message; + + $message = preg_replace('/\b(access_token|refresh_token|client_secret|password)\b\s*[:=]\s*[^\s,;]+/i', '[REDACTED_SECRET]', $message) ?? $message; + $message = preg_replace('/"(access_token|refresh_token|client_secret|password)"\s*:\s*"[^"]*"/i', '"[REDACTED]":"[REDACTED]"', $message) ?? $message; + + $message = preg_replace('/\b[A-Za-z0-9\-\._~\+\/]{64,}\b/', '[REDACTED]', $message) ?? $message; + + $message = str_ireplace( + ['client_secret', 'access_token', 'refresh_token', 'authorization', 'bearer '], + '[REDACTED]', + $message, + ); + + $message = trim($message); + + return $message === '' ? '—' : substr($message, 0, 240); + } + + private static function sanitizeShortString(mixed $value, ?string $fallback): ?string + { + if (! is_string($value)) { + return $fallback; + } + + $value = trim($value); + + if ($value === '') { + return $fallback; + } + + if (self::containsForbiddenKeySubstring($value)) { + return $fallback; + } + + return substr($value, 0, 200); + } + + private static function sanitizeValueString(string $value): ?string + { + $value = trim($value); + + if ($value === '') { + return null; + } + + if (preg_match('/\bBearer\s+[A-Za-z0-9\-\._~\+\/]+=*\b/i', $value)) { + return null; + } + + if (strlen($value) > 512) { + return null; + } + + if (preg_match('/\b[A-Za-z0-9\-\._~\+\/]{128,}\b/', $value)) { + return null; + } + + $lower = strtolower($value); + foreach (self::FORBIDDEN_KEY_SUBSTRINGS as $needle) { + if (str_contains($lower, $needle)) { + return null; + } + } + + return $value; + } + + private static function containsForbiddenKeySubstring(string $value): bool + { + $lower = strtolower($value); + + foreach (self::FORBIDDEN_KEY_SUBSTRINGS as $needle) { + if (str_contains($lower, $needle)) { + return true; + } + } + + return false; + } +} diff --git a/app/Support/Verification/VerificationReportSchema.php b/app/Support/Verification/VerificationReportSchema.php new file mode 100644 index 0000000..f6648d5 --- /dev/null +++ b/app/Support/Verification/VerificationReportSchema.php @@ -0,0 +1,235 @@ +|null + */ + public static function normalizeReport(mixed $report): ?array + { + if (! is_array($report)) { + return null; + } + + if (! self::isValidReport($report)) { + return null; + } + + return $report; + } + + /** + * @param array $report + */ + public static function isValidReport(array $report): bool + { + $schemaVersion = self::schemaVersion($report); + if ($schemaVersion === null || ! self::isSupportedSchemaVersion($schemaVersion)) { + return false; + } + + if (! self::isNonEmptyString($report['flow'] ?? null)) { + return false; + } + + if (! self::isIsoDateTimeString($report['generated_at'] ?? null)) { + return false; + } + + if (array_key_exists('identity', $report) && ! is_array($report['identity'])) { + return false; + } + + $summary = $report['summary'] ?? null; + if (! is_array($summary)) { + return false; + } + + $overall = $summary['overall'] ?? null; + if (! is_string($overall) || ! in_array($overall, VerificationReportOverall::values(), true)) { + return false; + } + + $counts = $summary['counts'] ?? null; + if (! is_array($counts)) { + return false; + } + + foreach (['total', 'pass', 'fail', 'warn', 'skip', 'running'] as $key) { + if (! self::isNonNegativeInt($counts[$key] ?? null)) { + return false; + } + } + + $checks = $report['checks'] ?? null; + if (! is_array($checks)) { + return false; + } + + foreach ($checks as $check) { + if (! is_array($check) || ! self::isValidCheckResult($check)) { + return false; + } + } + + return true; + } + + /** + * @param array $report + */ + public static function schemaVersion(array $report): ?string + { + $candidate = $report['schema_version'] ?? null; + + if (! is_string($candidate)) { + return null; + } + + $candidate = trim($candidate); + + if ($candidate === '') { + return null; + } + + if (! preg_match('/^\d+\.\d+\.\d+$/', $candidate)) { + return null; + } + + return $candidate; + } + + public static function isSupportedSchemaVersion(string $schemaVersion): bool + { + $parts = explode('.', $schemaVersion, 3); + + if (count($parts) !== 3) { + return false; + } + + $major = (int) $parts[0]; + + return $major === 1; + } + + /** + * @param array $check + */ + private static function isValidCheckResult(array $check): bool + { + if (! self::isNonEmptyString($check['key'] ?? null)) { + return false; + } + + if (! self::isNonEmptyString($check['title'] ?? null)) { + return false; + } + + $status = $check['status'] ?? null; + if (! is_string($status) || ! in_array($status, VerificationCheckStatus::values(), true)) { + return false; + } + + $severity = $check['severity'] ?? null; + if (! is_string($severity) || ! in_array($severity, VerificationCheckSeverity::values(), true)) { + return false; + } + + if (! is_bool($check['blocking'] ?? null)) { + return false; + } + + if (! self::isNonEmptyString($check['reason_code'] ?? null)) { + return false; + } + + if (! self::isNonEmptyString($check['message'] ?? null)) { + return false; + } + + $evidence = $check['evidence'] ?? null; + if (! is_array($evidence)) { + return false; + } + + foreach ($evidence as $pointer) { + if (! is_array($pointer) || ! self::isValidEvidencePointer($pointer)) { + return false; + } + } + + $nextSteps = $check['next_steps'] ?? null; + if (! is_array($nextSteps)) { + return false; + } + + foreach ($nextSteps as $step) { + if (! is_array($step) || ! self::isValidNextStep($step)) { + return false; + } + } + + return true; + } + + /** + * @param array $pointer + */ + private static function isValidEvidencePointer(array $pointer): bool + { + if (! self::isNonEmptyString($pointer['kind'] ?? null)) { + return false; + } + + $value = $pointer['value'] ?? null; + + return is_int($value) || self::isNonEmptyString($value); + } + + /** + * @param array $step + */ + private static function isValidNextStep(array $step): bool + { + if (! self::isNonEmptyString($step['label'] ?? null)) { + return false; + } + + if (! self::isNonEmptyString($step['url'] ?? null)) { + return false; + } + + return true; + } + + private static function isNonEmptyString(mixed $value): bool + { + return is_string($value) && trim($value) !== ''; + } + + private static function isNonNegativeInt(mixed $value): bool + { + return is_int($value) && $value >= 0; + } + + private static function isIsoDateTimeString(mixed $value): bool + { + if (! self::isNonEmptyString($value)) { + return false; + } + + try { + new DateTimeImmutable((string) $value); + + return true; + } catch (\Throwable) { + return false; + } + } +} diff --git a/app/Support/Verification/VerificationReportWriter.php b/app/Support/Verification/VerificationReportWriter.php new file mode 100644 index 0000000..2ea3ba6 --- /dev/null +++ b/app/Support/Verification/VerificationReportWriter.php @@ -0,0 +1,343 @@ + + */ + private const array BASELINE_REASON_CODES = [ + 'ok', + 'not_applicable', + 'missing_configuration', + 'permission_denied', + 'authentication_failed', + 'throttled', + 'dependency_unreachable', + 'invalid_state', + 'unknown_error', + ]; + + /** + * @param array> $checks + * @param array $identity + * @return array + */ + public static function write(OperationRun $run, array $checks, array $identity = []): array + { + $flow = is_string($run->type) && trim($run->type) !== '' ? (string) $run->type : 'unknown'; + + $report = self::build($flow, $checks, $identity); + $report = VerificationReportSanitizer::sanitizeReport($report); + + if (! VerificationReportSchema::isValidReport($report)) { + $report = VerificationReportSanitizer::sanitizeReport(self::buildFallbackReport($flow)); + } + + $context = is_array($run->context) ? $run->context : []; + $context['verification_report'] = $report; + + $run->update(['context' => $context]); + + return $report; + } + + /** + * @param array> $checks + * @param array $identity + * @return array + */ + public static function build(string $flow, array $checks, array $identity = []): array + { + $flow = trim($flow); + $flow = $flow !== '' ? $flow : 'unknown'; + + $normalizedChecks = []; + + foreach ($checks as $check) { + if (! is_array($check)) { + continue; + } + + $normalizedChecks[] = self::normalizeCheckResult($check); + } + + $counts = self::deriveCounts($normalizedChecks); + + $report = [ + 'schema_version' => VerificationReportSchema::CURRENT_SCHEMA_VERSION, + 'flow' => $flow, + 'generated_at' => now()->toISOString(), + 'summary' => [ + 'overall' => self::deriveOverall($normalizedChecks, $counts), + 'counts' => $counts, + ], + 'checks' => $normalizedChecks, + ]; + + if ($identity !== []) { + $report['identity'] = $identity; + } + + return $report; + } + + /** + * @return array + */ + private static function buildFallbackReport(string $flow): array + { + return [ + 'schema_version' => VerificationReportSchema::CURRENT_SCHEMA_VERSION, + 'flow' => $flow !== '' ? $flow : 'unknown', + 'generated_at' => now()->toISOString(), + 'summary' => [ + 'overall' => VerificationReportOverall::NeedsAttention->value, + 'counts' => [ + 'total' => 0, + 'pass' => 0, + 'fail' => 0, + 'warn' => 0, + 'skip' => 0, + 'running' => 0, + ], + ], + 'checks' => [], + ]; + } + + /** + * @param array $check + * @return array{ + * key: string, + * title: string, + * status: string, + * severity: string, + * blocking: bool, + * reason_code: string, + * message: string, + * evidence: array, + * next_steps: array + * } + */ + private static function normalizeCheckResult(array $check): array + { + $key = self::normalizeNonEmptyString($check['key'] ?? null, fallback: 'unknown_check'); + $title = self::normalizeNonEmptyString($check['title'] ?? null, fallback: 'Check'); + + return [ + 'key' => $key, + 'title' => $title, + 'status' => self::normalizeCheckStatus($check['status'] ?? null), + 'severity' => self::normalizeCheckSeverity($check['severity'] ?? null), + 'blocking' => is_bool($check['blocking'] ?? null) ? (bool) $check['blocking'] : false, + 'reason_code' => self::normalizeReasonCode($check['reason_code'] ?? null), + 'message' => self::normalizeNonEmptyString($check['message'] ?? null, fallback: '—'), + 'evidence' => self::normalizeEvidence($check['evidence'] ?? null), + 'next_steps' => self::normalizeNextSteps($check['next_steps'] ?? null), + ]; + } + + private static function normalizeCheckStatus(mixed $status): string + { + if (! is_string($status)) { + return VerificationCheckStatus::Fail->value; + } + + $status = strtolower(trim($status)); + + return in_array($status, VerificationCheckStatus::values(), true) + ? $status + : VerificationCheckStatus::Fail->value; + } + + private static function normalizeCheckSeverity(mixed $severity): string + { + if (! is_string($severity)) { + return VerificationCheckSeverity::Info->value; + } + + $severity = strtolower(trim($severity)); + + return in_array($severity, VerificationCheckSeverity::values(), true) + ? $severity + : VerificationCheckSeverity::Info->value; + } + + private static function normalizeReasonCode(mixed $reasonCode): string + { + if (! is_string($reasonCode)) { + return 'unknown_error'; + } + + $reasonCode = strtolower(trim($reasonCode)); + + if ($reasonCode === '') { + return 'unknown_error'; + } + + if (str_starts_with($reasonCode, 'ext.')) { + return $reasonCode; + } + + $reasonCode = match ($reasonCode) { + 'graph_throttled' => 'throttled', + 'graph_timeout', 'provider_outage' => 'dependency_unreachable', + 'provider_auth_failed' => 'authentication_failed', + 'validation_error', 'conflict_detected' => 'invalid_state', + 'unknown' => 'unknown_error', + default => $reasonCode, + }; + + return in_array($reasonCode, self::BASELINE_REASON_CODES, true) ? $reasonCode : 'unknown_error'; + } + + /** + * @return array + */ + private static function normalizeEvidence(mixed $evidence): array + { + if (! is_array($evidence)) { + return []; + } + + $normalized = []; + + foreach ($evidence as $pointer) { + if (! is_array($pointer)) { + continue; + } + + $kind = self::normalizeNonEmptyString($pointer['kind'] ?? null, fallback: null); + $value = $pointer['value'] ?? null; + + if ($kind === null) { + continue; + } + + if (! is_int($value) && ! is_string($value)) { + continue; + } + + if (is_string($value) && trim($value) === '') { + continue; + } + + $normalized[] = [ + 'kind' => $kind, + 'value' => is_int($value) ? $value : trim($value), + ]; + } + + return $normalized; + } + + /** + * @return array + */ + private static function normalizeNextSteps(mixed $steps): array + { + if (! is_array($steps)) { + return []; + } + + $normalized = []; + + foreach ($steps as $step) { + if (! is_array($step)) { + continue; + } + + $label = self::normalizeNonEmptyString($step['label'] ?? null, fallback: null); + $url = self::normalizeNonEmptyString($step['url'] ?? null, fallback: null); + + if ($label === null || $url === null) { + continue; + } + + $normalized[] = [ + 'label' => $label, + 'url' => $url, + ]; + } + + return $normalized; + } + + /** + * @param array $checks + * @return array{total: int, pass: int, fail: int, warn: int, skip: int, running: int} + */ + private static function deriveCounts(array $checks): array + { + $counts = [ + 'total' => count($checks), + 'pass' => 0, + 'fail' => 0, + 'warn' => 0, + 'skip' => 0, + 'running' => 0, + ]; + + foreach ($checks as $check) { + $status = $check['status'] ?? null; + + if (! is_string($status) || ! array_key_exists($status, $counts)) { + continue; + } + + $counts[$status] += 1; + } + + return $counts; + } + + /** + * @param array $checks + * @param array{total: int, pass: int, fail: int, warn: int, skip: int, running: int} $counts + */ + private static function deriveOverall(array $checks, array $counts): string + { + if (($counts['running'] ?? 0) > 0) { + return VerificationReportOverall::Running->value; + } + + if (($counts['total'] ?? 0) === 0) { + return VerificationReportOverall::NeedsAttention->value; + } + + foreach ($checks as $check) { + if (($check['status'] ?? null) === VerificationCheckStatus::Fail->value && ($check['blocking'] ?? false) === true) { + return VerificationReportOverall::Blocked->value; + } + } + + if (($counts['fail'] ?? 0) > 0 || ($counts['warn'] ?? 0) > 0) { + return VerificationReportOverall::NeedsAttention->value; + } + + return VerificationReportOverall::Ready->value; + } + + private static function normalizeNonEmptyString(mixed $value, ?string $fallback): ?string + { + if (! is_string($value)) { + return $fallback; + } + + $value = trim($value); + + if ($value === '') { + return $fallback; + } + + return $value; + } +} diff --git a/resources/views/filament/components/verification-report-viewer.blade.php b/resources/views/filament/components/verification-report-viewer.blade.php new file mode 100644 index 0000000..b640b52 --- /dev/null +++ b/resources/views/filament/components/verification-report-viewer.blade.php @@ -0,0 +1,178 @@ +@php + $report = isset($getState) ? $getState() : ($report ?? null); + $report = is_array($report) ? $report : null; + + $summary = $report['summary'] ?? null; + $summary = is_array($summary) ? $summary : null; + + $counts = $summary['counts'] ?? null; + $counts = is_array($counts) ? $counts : []; + + $checks = $report['checks'] ?? null; + $checks = is_array($checks) ? $checks : []; +@endphp + +
+ @if ($report === null || $summary === null) +
+
+ Verification report unavailable +
+
+ This run doesn’t have a report yet. If it’s still running, refresh in a moment. If it already completed, start verification again. +
+
+ @else + @php + $overallSpec = \App\Support\Badges\BadgeRenderer::spec( + \App\Support\Badges\BadgeDomain::VerificationReportOverall, + $summary['overall'] ?? null, + ); + @endphp + +
+ + {{ $overallSpec->label }} + + + + {{ (int) ($counts['total'] ?? 0) }} total + + + {{ (int) ($counts['pass'] ?? 0) }} pass + + + {{ (int) ($counts['fail'] ?? 0) }} fail + + + {{ (int) ($counts['warn'] ?? 0) }} warn + + + {{ (int) ($counts['skip'] ?? 0) }} skip + + + {{ (int) ($counts['running'] ?? 0) }} running + +
+ + @if ($checks === []) +
+ No checks found in this report. Start verification again to generate a fresh report. +
+ @else +
+ @foreach ($checks as $check) + @php + $check = is_array($check) ? $check : []; + + $title = $check['title'] ?? 'Check'; + $title = is_string($title) && trim($title) !== '' ? $title : 'Check'; + + $message = $check['message'] ?? null; + $message = is_string($message) && trim($message) !== '' ? $message : null; + + $statusSpec = \App\Support\Badges\BadgeRenderer::spec( + \App\Support\Badges\BadgeDomain::VerificationCheckStatus, + $check['status'] ?? null, + ); + + $severitySpec = \App\Support\Badges\BadgeRenderer::spec( + \App\Support\Badges\BadgeDomain::VerificationCheckSeverity, + $check['severity'] ?? null, + ); + + $evidence = $check['evidence'] ?? []; + $evidence = is_array($evidence) ? $evidence : []; + + $nextSteps = $check['next_steps'] ?? []; + $nextSteps = is_array($nextSteps) ? $nextSteps : []; + @endphp + +
+ +
+
+ {{ $title }} +
+ @if ($message) +
+ {{ $message }} +
+ @endif +
+ +
+ + {{ $severitySpec->label }} + + + {{ $statusSpec->label }} + +
+ + + @if ($evidence !== [] || $nextSteps !== []) +
+ @if ($evidence !== []) +
+
+ Evidence +
+
    + @foreach ($evidence as $pointer) + @php + $pointer = is_array($pointer) ? $pointer : []; + $kind = $pointer['kind'] ?? null; + $value = $pointer['value'] ?? null; + @endphp + + @if (is_string($kind) && $kind !== '' && (is_string($value) || is_int($value))) +
  • + {{ $kind }}: + {{ is_int($value) ? $value : $value }} +
    • + @endif + @endforeach +
+
+ @endif + + @if ($nextSteps !== []) +
+
+ Next steps +
+
    + @foreach ($nextSteps as $step) + @php + $step = is_array($step) ? $step : []; + $label = $step['label'] ?? null; + $url = $step['url'] ?? null; + $isExternal = is_string($url) && (str_starts_with($url, 'http://') || str_starts_with($url, 'https://')); + @endphp + + @if (is_string($label) && $label !== '' && is_string($url) && $url !== '') +
  • + + {{ $label }} + +
    • + @endif + @endforeach +
+
+ @endif +
+ @endif +
+ @endforeach +
+ @endif + @endif +
diff --git a/specs/074-verification-checklist/checklists/requirements.md b/specs/074-verification-checklist/checklists/requirements.md new file mode 100644 index 0000000..f6e7a69 --- /dev/null +++ b/specs/074-verification-checklist/checklists/requirements.md @@ -0,0 +1,34 @@ +# Specification Quality Checklist: Verification Checklist Framework (Enterprise-Ready) + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-02-03 +**Feature**: [specs/074-verification-checklist/spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Validation pass (2026-02-03): Spec avoids framework specifics and focuses on contract + UX outcomes. Next step is planning to translate these requirements into a minimal set of deliverables (report schema, viewer, authorization semantics, audit events, and adoption points). diff --git a/specs/074-verification-checklist/contracts/examples/fail.json b/specs/074-verification-checklist/contracts/examples/fail.json new file mode 100644 index 0000000..f113920 --- /dev/null +++ b/specs/074-verification-checklist/contracts/examples/fail.json @@ -0,0 +1,47 @@ +{ + "schema_version": "1.0.0", + "flow": "provider.connection.check", + "generated_at": "2026-02-03T22:00:00Z", + "identity": { + "provider_connection_id": 123 + }, + "summary": { + "overall": "blocked", + "counts": { + "total": 2, + "pass": 1, + "fail": 1, + "warn": 0, + "skip": 0, + "running": 0 + } + }, + "checks": [ + { + "key": "provider_connection.token_acquisition", + "title": "Token acquisition works", + "status": "fail", + "severity": "high", + "blocking": true, + "reason_code": "authentication_failed", + "message": "The app cannot acquire a token with the configured credentials.", + "evidence": [ + { "kind": "provider_connection_id", "value": 123 } + ], + "next_steps": [ + { "label": "Review connection credentials", "url": "/admin/provider-connections/123/edit" } + ] + }, + { + "key": "provider_connection.permissions", + "title": "Required permissions are granted", + "status": "pass", + "severity": "info", + "blocking": false, + "reason_code": "ok", + "message": "The configured app permissions meet the required baseline.", + "evidence": [], + "next_steps": [] + } + ] +} diff --git a/specs/074-verification-checklist/contracts/examples/pass.json b/specs/074-verification-checklist/contracts/examples/pass.json new file mode 100644 index 0000000..42c382c --- /dev/null +++ b/specs/074-verification-checklist/contracts/examples/pass.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0.0", + "flow": "provider.connection.check", + "generated_at": "2026-02-03T22:00:00Z", + "summary": { + "overall": "ready", + "counts": { + "total": 1, + "pass": 1, + "fail": 0, + "warn": 0, + "skip": 0, + "running": 0 + } + }, + "checks": [ + { + "key": "provider_connection.health", + "title": "Provider connection is healthy", + "status": "pass", + "severity": "info", + "blocking": false, + "reason_code": "ok", + "message": "The provider connection passed all required health checks.", + "evidence": [], + "next_steps": [] + } + ] +} diff --git a/specs/074-verification-checklist/contracts/examples/running.json b/specs/074-verification-checklist/contracts/examples/running.json new file mode 100644 index 0000000..8c12cd6 --- /dev/null +++ b/specs/074-verification-checklist/contracts/examples/running.json @@ -0,0 +1,51 @@ +{ + "schema_version": "1.0.0", + "flow": "provider.connection.check", + "generated_at": "2026-02-03T22:00:00Z", + "summary": { + "overall": "running", + "counts": { + "total": 3, + "pass": 1, + "fail": 0, + "warn": 0, + "skip": 0, + "running": 2 + } + }, + "checks": [ + { + "key": "provider_connection.token_acquisition", + "title": "Token acquisition works", + "status": "running", + "severity": "info", + "blocking": false, + "reason_code": "ok", + "message": "Check is currently running.", + "evidence": [], + "next_steps": [] + }, + { + "key": "provider_connection.permissions", + "title": "Required permissions are granted", + "status": "running", + "severity": "info", + "blocking": false, + "reason_code": "ok", + "message": "Check is currently running.", + "evidence": [], + "next_steps": [] + }, + { + "key": "provider_connection.health", + "title": "Provider connection is healthy", + "status": "pass", + "severity": "info", + "blocking": false, + "reason_code": "ok", + "message": "The provider connection passed all required health checks.", + "evidence": [], + "next_steps": [] + } + ] +} diff --git a/specs/074-verification-checklist/contracts/examples/warn.json b/specs/074-verification-checklist/contracts/examples/warn.json new file mode 100644 index 0000000..2795bfb --- /dev/null +++ b/specs/074-verification-checklist/contracts/examples/warn.json @@ -0,0 +1,42 @@ +{ + "schema_version": "1.0.0", + "flow": "provider.connection.check", + "generated_at": "2026-02-03T22:00:00Z", + "summary": { + "overall": "needs_attention", + "counts": { + "total": 2, + "pass": 1, + "fail": 0, + "warn": 1, + "skip": 0, + "running": 0 + } + }, + "checks": [ + { + "key": "provider_connection.optional_metadata", + "title": "Optional metadata is present", + "status": "warn", + "severity": "medium", + "blocking": false, + "reason_code": "missing_configuration", + "message": "Some optional metadata is missing; this may reduce diagnostics quality.", + "evidence": [], + "next_steps": [ + { "label": "Open provider connection settings", "url": "/admin/provider-connections" } + ] + }, + { + "key": "provider_connection.health", + "title": "Provider connection is healthy", + "status": "pass", + "severity": "info", + "blocking": false, + "reason_code": "ok", + "message": "The provider connection passed all required health checks.", + "evidence": [], + "next_steps": [] + } + ] +} diff --git a/specs/074-verification-checklist/contracts/reason-codes.md b/specs/074-verification-checklist/contracts/reason-codes.md new file mode 100644 index 0000000..5496b4d --- /dev/null +++ b/specs/074-verification-checklist/contracts/reason-codes.md @@ -0,0 +1,26 @@ +# Reason Codes (074) + +This file defines the baseline `reason_code` taxonomy for verification check results. + +## Rules + +- Reason codes are **stable** and **machine-readable**. +- New codes must be appended (avoid renames) to keep support and automation stable. +- Flow/check-specific codes must use the reserved namespace: `ext.*`. + +## Baseline Codes (v1) + +- `ok` — Check passed. +- `not_applicable` — Check skipped because it doesn’t apply to this identity/scope. +- `missing_configuration` — Required config is absent. +- `permission_denied` — Insufficient permissions / consent missing. +- `authentication_failed` — Token acquisition or auth precondition failed. +- `throttled` — Remote dependency throttled (e.g., 429/503) and check could not complete. +- `dependency_unreachable` — Remote dependency unavailable. +- `invalid_state` — Local model state conflicts with required preconditions. +- `unknown_error` — Failure could not be classified. + +## Reserved Extension Namespace + +- `ext..` — Flow-specific extensions. + - Example: `ext.managed_tenant_onboarding.role_mapping_missing` diff --git a/specs/074-verification-checklist/contracts/verification-report.schema.json b/specs/074-verification-checklist/contracts/verification-report.schema.json new file mode 100644 index 0000000..e2e9938 --- /dev/null +++ b/specs/074-verification-checklist/contracts/verification-report.schema.json @@ -0,0 +1,128 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://tenantpilot.local/contracts/verification-report.schema.json", + "title": "VerificationReport", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "flow", + "generated_at", + "summary", + "checks" + ], + "properties": { + "schema_version": { + "type": "string", + "description": "Version of the verification report schema (SemVer)." + }, + "flow": { + "type": "string", + "description": "Verification flow identifier (v1 aligns with OperationRun.type)." + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "identity": { + "type": "object", + "description": "Scope identifiers for what is being verified.", + "additionalProperties": true + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["overall", "counts"], + "properties": { + "overall": { + "type": "string", + "enum": ["ready", "needs_attention", "blocked", "running"], + "description": "Overall state derived from check results." + }, + "counts": { + "type": "object", + "additionalProperties": false, + "required": ["total", "pass", "fail", "warn", "skip", "running"], + "properties": { + "total": {"type": "integer", "minimum": 0}, + "pass": {"type": "integer", "minimum": 0}, + "fail": {"type": "integer", "minimum": 0}, + "warn": {"type": "integer", "minimum": 0}, + "skip": {"type": "integer", "minimum": 0}, + "running": {"type": "integer", "minimum": 0} + } + } + } + }, + "checks": { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/$defs/CheckResult"} + } + }, + "$defs": { + "CheckResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "title", + "status", + "severity", + "blocking", + "reason_code", + "message", + "evidence", + "next_steps" + ], + "properties": { + "key": {"type": "string"}, + "title": {"type": "string"}, + "status": { + "type": "string", + "enum": ["pass", "fail", "warn", "skip", "running"] + }, + "severity": { + "type": "string", + "enum": ["info", "low", "medium", "high", "critical"] + }, + "blocking": {"type": "boolean"}, + "reason_code": {"type": "string"}, + "message": {"type": "string"}, + "evidence": { + "type": "array", + "items": {"$ref": "#/$defs/EvidencePointer"} + }, + "next_steps": { + "type": "array", + "description": "Navigation-only CTAs (links) in v1.", + "items": {"$ref": "#/$defs/NextStep"} + } + } + }, + "EvidencePointer": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "value"], + "properties": { + "kind": {"type": "string"}, + "value": { + "description": "Safe pointer value (ID/masked string/hash).", + "oneOf": [ + {"type": "integer"}, + {"type": "string"} + ] + } + } + }, + "NextStep": { + "type": "object", + "additionalProperties": false, + "required": ["label", "url"], + "properties": { + "label": {"type": "string"}, + "url": {"type": "string"} + } + } + } +} diff --git a/specs/074-verification-checklist/data-model.md b/specs/074-verification-checklist/data-model.md new file mode 100644 index 0000000..0b380d6 --- /dev/null +++ b/specs/074-verification-checklist/data-model.md @@ -0,0 +1,61 @@ +# Data Model: Verification Checklist Framework (074) + +## Overview + +This feature introduces a *versioned verification report document* attached to an existing `OperationRun`. +No new database tables are required for v1. + +## Existing Entities Used + +### OperationRun (`operation_runs`) + +Selected fields: +- `id` +- `tenant_id` +- `user_id` +- `type` (used as the verification flow identifier) +- `status` (`queued` | `running` | `completed`) +- `outcome` (`pending` | `succeeded` | `failed`) +- `summary_counts` (JSONB) +- `failure_summary` (JSONB) +- `context` (JSONB) +- `started_at`, `completed_at` + +Idempotency: +- DB-enforced dedupe for active runs via partial unique index on `(tenant_id, run_identity_hash)` where `status IN ('queued','running')`. + +## New Logical Data (stored inside OperationRun context) + +### VerificationReport (`operation_runs.context.verification_report`) + +- Stored as JSON in `context` under `verification_report`. +- Versioned by `schema_version`. +- Rendered DB-only (no external calls during view). + +High-level shape (see `contracts/verification-report.schema.json` for the canonical contract): +- `schema_version` +- `flow` (identifier; for v1 this can align with `operation_runs.type`) +- `identity` (scope identifiers such as `tenant_id`, `provider_connection_id`, etc.) +- `generated_at` +- `summary` (counts, overall state) +- `checks[]` (check results) + +### CheckResult (within `checks[]`) + +- `key`, `title` +- `status`: `pass|fail|warn|skip|running` +- `severity`: `info|low|medium|high|critical` +- `blocking`: boolean +- `reason_code` +- `message` +- `evidence[]`: safe pointers only +- `next_steps[]`: links only in v1 + +## Audit + +Verification start and completion are recorded in `audit_logs` using stable `action` identifiers (via `App\Support\Audit\AuditActionId`). Metadata is minimal and sanitized. + +## Notes / Constraints + +- Viewer must be DB-only: rendering the report must not dispatch jobs or perform HTTP. +- Evidence must be redacted/safe: no secrets/tokens/payload dumps in stored or rendered report. diff --git a/specs/074-verification-checklist/plan.md b/specs/074-verification-checklist/plan.md new file mode 100644 index 0000000..0e03c06 --- /dev/null +++ b/specs/074-verification-checklist/plan.md @@ -0,0 +1,127 @@ +# Implementation Plan: Verification Checklist Framework (Enterprise-Ready) + +**Branch**: `074-verification-checklist` | **Date**: 2026-02-03 | **Spec**: [spec.md](./spec.md) +**Input**: Feature specification from `/specs/074-verification-checklist/spec.md` + +**Note**: This file is generated from the plan template and then filled in by `/speckit.plan` workflow steps. + +## Summary + +- Introduce a versioned “verification report” contract that can be attached to an existing `OperationRun` and rendered consistently across multiple flows. +- Provide a reusable, DB-only report viewer (no outbound calls during render/hydration/poll) that presents summary + per-check statuses + safe evidence pointers + navigation-only next steps. +- Enforce enterprise semantics: stable reason codes, strict evidence redaction, deterministic active-run dedupe, and capability-first authorization aligned with RBAC-UX (non-members 404; members missing start capability 403). +- Emit audit events for verification start + completion using stable action identifiers with redacted metadata. + +## Technical Context + +**Language/Version**: PHP 8.4 (Laravel 12) +**Primary Dependencies**: Laravel 12 + Filament v5 + Livewire v4 +**Storage**: PostgreSQL (Sail) with JSONB (`operation_runs.context`) +**Testing**: Pest (PHPUnit) +**Target Platform**: Web application (Sail/Docker locally; container deploy via Dokploy) +**Project Type**: web +**Performance Goals**: Verification viewer renders fast from DB-only JSON (typical report ≤ 50 checks) +**Constraints**: Viewer is read-only and must not trigger any outbound HTTP or job dispatch; evidence must not contain secrets/tokens/payloads; new status-like badges must use centralized BADGE-001 mapping. +**Scale/Scope**: Multiple tenants/workspaces; many runs over time; verification used in onboarding + provider ops + future readiness flows + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +- Inventory-first, snapshots-second: PASS (this feature is run/report UX; no inventory semantics changed). +- Read/write separation: PASS (viewer is read-only; start surfaces enqueue-only and already follow `OperationRun` patterns). +- Graph contract path: PASS (viewer performs no Graph calls; verification execution remains in queued jobs that already follow provider gateway patterns). +- Deterministic capabilities: PASS (start/view gates reference the existing capability registry; no role-string checks). +- RBAC-UX: PASS (non-member tenant access is 404; member-but-missing-capability is 403; server-side gates enforce mutations/starts). +- Run observability: PASS (verification is represented as `OperationRun`; active-run dedupe enforced by the existing partial unique index on `(tenant_id, run_identity_hash)` for active statuses). +- Data minimization: PASS (report evidence constrained to safe pointers; audit metadata redacted; no secrets in stored report). +- Badge semantics (BADGE-001): PASS (plan includes adding a centralized badge domain for check statuses/severity; no ad-hoc UI mappings). + +## Project Structure + +### Documentation (this feature) + +```text +specs/074-verification-checklist/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ # Phase 1 output +└── tasks.md # Phase 2 output (/speckit.tasks) +``` + +### Source Code (repository root) + +```text +app/ +├── Filament/ +├── Jobs/ +├── Models/ +├── Policies/ +├── Services/ +└── Support/ + +config/ +database/ +resources/ +routes/ +tests/ +``` + +**Structure Decision**: Single Laravel web application with Filament admin panel. The framework is implemented as: +- contract + helpers under `app/Support/**` +- report writer invoked from queued jobs under `app/Jobs/**` / `app/Services/**` +- viewer UI as Filament schema components and Blade views under `app/Filament/**` and `resources/views/filament/**` +- authorization via existing capabilities/gates/policies + +## Complexity Tracking + +No constitution violations required for this feature. + +## Phase 0 — Research (output: `research.md`) + +See: [research.md](./research.md) + +Goals: +- Confirm the canonical storage location for the report (DB-only render) using existing `operation_runs.context` JSONB. +- Confirm active-run dedupe behavior and ensure it matches the spec’s “dedupe while active only” requirement. +- Confirm the correct approach for status-like UI badges in Filament (BADGE-001), so the viewer doesn’t introduce ad-hoc mappings. +- Confirm the existing audit logger + redaction utilities and define stable action IDs for verification completion. + +## Phase 1 — Design & Contracts (outputs: `data-model.md`, `contracts/`, `quickstart.md`) + +See: +- [data-model.md](./data-model.md) +- [contracts/](./contracts/) +- [quickstart.md](./quickstart.md) + +Design focus: +- Report schema: versioned JSON document (checks + counts + timestamps + next steps) stored in `operation_runs.context.verification_report`. +- Reason codes: baseline set + reserved `ext.*` namespace. +- Evidence redaction: strict sanitizer so reports never store or render secrets/tokens/payloads. +- Viewer: reusable Filament view entry / component that renders summary + per-check details without any outbound calls. +- Authorization: view allowed for tenant-scoped members; start requires capability; non-member access is deny-as-not-found. +- Auditing: start + completion events logged with minimal redacted metadata. + +## Phase 2 — Implementation Outline (tasks created in `/speckit.tasks`) + +- Contract: create a canonical report schema (JSON Schema + example) and a small baseline reason-code list. +- Writer: add a `VerificationReportWriter` (or equivalent) that normalizes check results, enforces redaction rules, and writes the report into `OperationRun->context`. +- Viewer: add a reusable Filament UI renderer (Blade view + helper) that displays summary, counts, statuses, and next steps (links-only). +- Integration points: + - Show the verification report section in Monitoring → Operations run detail when present. + - Embed the same viewer in onboarding and provider connection verification flows. +- RBAC & UX: + - Enforce “view vs start” split (view allowed for tenant members; start capability required). + - Preserve RBAC-UX semantics (non-members 404; members missing capability 403). +- Audit: + - Keep existing start audit event; add a completion audit event emitted when the verification run finalizes. +- Tests (Pest): + - Viewer is DB-only (Http::fake + render assertion). + - Evidence redaction rules (report contains no forbidden keys/values). + - Dedupe semantics reuse active run (leveraging the existing partial unique index behavior). + +## Constitution Check (Post-Design) + +Re-check result: PASS. Design artifacts keep verification viewing DB-only, align with run observability + dedupe, enforce RBAC-UX semantics, and centralize status badge mappings. diff --git a/specs/074-verification-checklist/quickstart.md b/specs/074-verification-checklist/quickstart.md new file mode 100644 index 0000000..68a94c9 --- /dev/null +++ b/specs/074-verification-checklist/quickstart.md @@ -0,0 +1,79 @@ +# Quickstart: Verification Checklist Framework (074) + +This quickstart explains how to *write* and *render* a verification report attached to an `OperationRun`. + +## 1) Writing a report (queued job / service) + +**Goal**: produce a `verification_report` JSON document and store it in `OperationRun->context`. + +Guidelines: +- Generate reports inside queued execution (not in a Filament page render). +- Keep evidence pointer-only (IDs/masked/hashes), never raw payloads or tokens. +- Keep next steps navigation-only in v1. + +Pseudo-code sketch: + +```php +$context = is_array($run->context) ? $run->context : []; + +$context['verification_report'] = [ + 'schema_version' => '1.0', + 'flow' => $run->type, + 'generated_at' => now('UTC')->toIso8601String(), + 'identity' => [ + 'tenant_id' => (int) $run->tenant_id, + 'provider_connection_id' => (int) data_get($run->context, 'provider_connection_id', 0), + ], + 'summary' => [ + 'overall' => 'needs_attention', + 'counts' => [ + 'total' => 5, + 'pass' => 3, + 'fail' => 2, + 'warn' => 0, + 'skip' => 0, + 'running' => 0, + ], + ], + 'checks' => [ + [ + 'key' => 'provider_connection.token_acquisition', + 'title' => 'Token acquisition works', + 'status' => 'fail', + 'severity' => 'high', + 'blocking' => true, + 'reason_code' => 'permission_denied', + 'message' => 'The app cannot acquire a token with the configured credentials.', + 'evidence' => [ + ['kind' => 'provider_connection_id', 'value' => (int) data_get($run->context, 'provider_connection_id')], + ], + 'next_steps' => [ + ['label' => 'Review connection credentials', 'url' => '/admin/...'], + ['label' => 'Microsoft docs: app permissions', 'url' => 'https://learn.microsoft.com/...'], + ], + ], + ], +]; + +$run->update(['context' => $context]); +``` + +## 2) Rendering the report (Filament, DB-only) + +Recommended integration points: +- Monitoring → Operations: in the `OperationRun` view page, show a “Verification report” section when `context.verification_report` exists. +- Flow pages (e.g., onboarding wizard): embed the same viewer component using the run ID stored in wizard state. + +**Hard requirement**: rendering must not trigger any outbound HTTP (no Graph calls, no jobs dispatched, no side effects). + +## 3) Authorization split + +- Viewing a report: allowed for tenant-scoped members. +- Starting verification: requires a specific capability. +- Non-members: deny-as-not-found (404) for tenant-scoped pages and actions. + +## 4) Tests to add + +- Viewer DB-only render test: `Http::fake()` + assert no requests during render. +- Evidence redaction test: report JSON contains none of `access_token`, `client_secret`, `Authorization`, bearer tokens, or raw payload dumps. +- Dedupe test: repeated starts while active reuse the same run. diff --git a/specs/074-verification-checklist/research.md b/specs/074-verification-checklist/research.md new file mode 100644 index 0000000..bacd150 --- /dev/null +++ b/specs/074-verification-checklist/research.md @@ -0,0 +1,86 @@ +# Research: Verification Checklist Framework (074) + +**Date**: 2026-02-03 +**Phase**: Phase 0 (Foundational Research) +**Status**: Complete + +--- + +## Decisions + +### D-001 — Canonical storage location for verification reports + +**Decision**: Store the verification report in `operation_runs.context.verification_report` (JSONB). + +**Rationale**: +- Monitoring pages must be DB-only at render time (constitution: Operations / Run Observability Standard). +- `OperationRun` is the canonical operational record; keeping the report attached avoids new tables/indexing for v1. +- The existing UI already renders `OperationRun.context` safely as JSON, so we can progressively enhance into a structured viewer. + +**Alternatives considered**: +- Dedicated `verification_reports` table: rejected for v1 to keep adoption lightweight; can be introduced later if querying/indexing becomes necessary. + +--- + +### D-002 — Idempotency / dedupe mechanism + +**Decision**: Use the existing `OperationRunService::ensureRunWithIdentity()` mechanism and the DB partial unique index on `(tenant_id, run_identity_hash)` for active statuses (`queued`, `running`). + +**Rationale**: +- This repo already enforces active-run dedupe at the DB level via `operation_runs_active_unique`. +- Matches the clarified spec policy: dedupe only while a run is active; completed runs allow a new run. + +**Alternatives considered**: +- Application-only locks/dedupe: rejected as non-race-safe. + +--- + +### D-003 — Flow identifier and identity scope + +**Decision**: Treat `OperationRun.type` as the primary flow identifier for the verification run, and keep additional flow details (wizard step, etc.) in `context`. + +**Rationale**: +- Existing operations already key UX semantics (labels, polling, related links) off `OperationRun.type`. +- Dedupe identity hashing already includes `type`, making flow part of the dedupe boundary. + +**Alternatives considered**: +- Separate `flow_id` column: rejected for v1 (schema change not required). + +--- + +### D-004 — Reason code taxonomy and extensions + +**Decision**: Maintain a small baseline set of cross-cutting reason codes, and reserve `ext.*` for flow/check-specific extensions. + +**Rationale**: +- Prevents brittle UI parsing and enables future automation. +- Keeps room for flow-specific details without polluting the baseline vocabulary. + +**Alternatives considered**: +- Free-form codes everywhere: rejected due to support/automation cost. + +--- + +### D-005 — Evidence policy (strict safe pointers) + +**Decision**: Evidence fields in check results are *strictly* structured safe pointers only (IDs, masked strings, hashes). No payloads, tokens, claims, headers, or full error bodies. + +**Rationale**: +- Aligns with constitution data-minimization and safe logging rules. +- Avoids accidentally persisting secrets inside run context. + +**Alternatives considered**: +- Storing raw error payloads: rejected for security and compliance risk. + +--- + +### D-006 — UI semantics for statuses and badges + +**Decision**: Render status-like values (check status, severity) via centralized badge semantics (BADGE-001), not ad-hoc mappings in feature pages. + +**Rationale**: +- Prevents drift in meaning/colors across the suite. +- Enables straightforward regression tests for new/changed status values. + +**Alternatives considered**: +- Inline color mapping inside a Blade view: rejected (violates BADGE-001). diff --git a/specs/074-verification-checklist/spec.md b/specs/074-verification-checklist/spec.md new file mode 100644 index 0000000..2b1d4cd --- /dev/null +++ b/specs/074-verification-checklist/spec.md @@ -0,0 +1,186 @@ +# Feature Specification: Verification Checklist Framework (Enterprise-Ready) + +**Feature Branch**: `074-verification-checklist` +**Created**: 2026-02-03 +**Status**: Draft +**Input**: User description: "Replace binary verification UX with a structured, reusable verification checklist attached to verification runs; DB-only viewing; enterprise semantics (reason codes, audit, idempotency, RBAC)." + +## Clarifications + +### Session 2026-02-03 + +- Q: What idempotency policy do we want for “Start verification”? → A: Dedupe only while a run is active (queued/running); once completed/failed, “Start verification” creates a new run. +- Q: Who should be allowed to view verification reports? → A: Any authenticated workspace member with access to the tenant scope may view reports; starting verification requires a separate capability. +- Q: What policy should we use for `reason_code` taxonomy? → A: Versioned central taxonomy with a small baseline set + reserved `ext.*` namespace for feature-specific extensions. +- Q: What’s the required evidence/redaction policy for `evidence` in check results? → A: Evidence is strictly structured safe pointers only (internal IDs, masked strings, hashes); never raw payloads, tokens, claims, headers, or full error bodies. +- Q: Should “Next steps” CTAs be links only, or can they trigger server-side actions? → A: Links only (navigation-only) in v1. + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - Operator sees what’s wrong (Priority: P1) + +As a workspace member onboarding or operating a managed tenant, I can run “Verify access” and see a structured checklist that clearly shows which checks passed, which failed, and what to do next. + +**Why this priority**: This is the primary value of verification: reduce ambiguity and enable fast, correct remediation. + +**Independent Test**: Seed a verification run with a report containing mixed outcomes and confirm the viewer renders an accurate summary, per-check status, and next steps without making any external calls. + +**Acceptance Scenarios**: + +1. **Given** a completed verification run with 2 failed checks, **When** I open the verification report viewer, **Then** I see an overall summary (“Needs attention” or “Blocked”), counts, and the two failed checks with actionable next steps. +2. **Given** a verification run that is still in progress, **When** I open the viewer, **Then** I see a “Running” state and partial results (if available) without errors. + +--- + +### User Story 2 - Deterministic starts (idempotency / dedupe) (Priority: P2) + +As an operator, if I click “Start verification” multiple times for the same tenant + provider connection + flow, the system behaves deterministically: it does not start duplicate active runs and guides me to the already-running run. + +**Why this priority**: Prevents confusing duplicates, reduces load, and makes support/debugging repeatable. + +**Independent Test**: Attempt to start verification twice for the same identity and assert that only one active run exists and the UI returns a consistent “already running” outcome. + +**Acceptance Scenarios**: + +1. **Given** an active verification run exists for the same identity, **When** I click “Start verification”, **Then** no duplicate run is started and I am directed to view the active run/report. +2. **Given** no active verification run exists (including when the most recent run is completed or failed), **When** I click “Start verification”, **Then** a new run starts and I can view its report as it progresses. + +--- + +### User Story 3 - Least-privilege and safe disclosure (Priority: P3) + +As a workspace member with access to the tenant scope (including read-only), I can view verification reports but cannot start verification unless I have the start capability. As a non-member, I cannot discover that a tenant or report exists. + +**Why this priority**: Verification data can leak operational posture; access must follow least-privilege and “deny-as-not-found” for non-members. + +**Independent Test**: Validate both authorization paths: read-only can view but cannot start; non-member receives a not-found response for all tenant-scoped verification routes. + +**Acceptance Scenarios**: + +1. **Given** I am a workspace member without the “start verification” capability, **When** I open the verification page, **Then** I can view past reports but the “Start verification” action is disabled and cannot be executed. +2. **Given** I am not a member of the workspace/tenant scope, **When** I attempt to access the verification report route, **Then** I receive a not-found response with no identifying hints. + +--- + +### Edge Cases + +- Report missing or malformed (e.g., run exists but report is absent or partial) → viewer shows a safe “Report unavailable” state and guidance. +- Unknown check keys or unknown reason codes (newer schema written by a newer verifier) → viewer degrades gracefully, still showing status/message/next steps when present. +- Large reports (near upper bound, e.g., 50 checks) → viewer remains responsive and summary counts remain correct. +- A run transitions from running → complete while the user is viewing → the viewer refreshes safely or the user can re-open without inconsistent states. +- Evidence contains unexpected fields → redaction rules prevent sensitive values from being displayed. + +## Out of Scope + +- Introducing a separate monitoring/observability platform beyond the existing run tracking and audit log. +- Any workflow that requires client-side handling of secrets. +- A full overhaul of onboarding wizards beyond replacing/embedding verification status with the checklist viewer. +- Provider job orchestration redesign unrelated to running verification checks. +- Server-side actions triggered directly from the checklist viewer (v1 is navigation-only). + +## Requirements *(mandatory)* + +**Constitution alignment (required):** If this feature introduces any external provider calls, any write/change behavior, +or any long-running/background work, the spec MUST describe safety gates (preview/confirmation/audit), tenant isolation, +run observability (run identity, visibility, and outcomes), and tests. If security-relevant DB-only actions intentionally +skip run tracking, the spec MUST describe the audit log entries. + +**Constitution alignment (RBAC-UX):** If this feature introduces or changes authorization behavior, the spec MUST: +- state which authorization plane(s) are involved (tenant-scoped admin area vs platform/system admin area), +- ensure any cross-plane access is deny-as-not-found (404), +- explicitly define 404 vs 403 semantics: + - non-member / not entitled to tenant scope → 404 (deny-as-not-found) + - member but missing capability → 403 +- describe how authorization is enforced server-side (Gates/Policies) for every mutation/operation-start/credential change, +- reference the canonical capability registry (no raw capability strings; no role-string checks in feature code), +- ensure global search is tenant-scoped and non-member-safe (no hints; inaccessible results treated as 404 semantics), +- ensure destructive-like actions require explicit user confirmation, +- include at least one positive and one negative authorization test, and note any RBAC regression tests added/updated. + +**Constitution alignment (OPS-EX-AUTH-001):** OIDC/SAML login handshakes may perform synchronous outbound HTTP (e.g., token exchange) +on `/auth/*` endpoints without an `OperationRun`. This MUST NOT be used for Monitoring/Operations pages. + +**Constitution alignment (BADGE-001):** If this feature changes status-like badges (status/outcome/severity/risk/availability/boolean), +the spec MUST describe how badge semantics stay centralized (no ad-hoc mappings) and which tests cover any new/changed values. + +### Functional Requirements + +- **FR-001 — Canonical verification report contract**: The system MUST generate a versioned “Verification Report” document for each verification run, including: schema version, flow identifier, identity/scope, generated timestamp, summary counts, and a list of check results. + +- **FR-002 — Check result contract**: Each check result MUST include: stable key, title, status (pass/fail/warn/skip/running), severity (info/low/medium/high/critical), blocking flag, reason code, human-readable message, safe evidence pointers, and one or more “next steps” actions (where applicable). + + Evidence MUST be strictly limited to structured safe pointers (internal IDs, masked strings, hashes) and MUST NOT contain raw payloads, tokens, claims, headers, or full error bodies. + + Next steps in v1 MUST be navigation-only (links to internal pages or external documentation) and MUST NOT trigger server-side actions. + +- **FR-003 — Stable reason code taxonomy**: The system MUST use stable, documented reason codes for failed/warned/skipped outcomes so that support, automation, and future UI changes remain consistent. + + The taxonomy MUST include a small baseline set of cross-cutting codes and MUST reserve an `ext.*` namespace for flow-specific or check-specific extensions. + +- **FR-004 — DB-only viewing**: Viewing a verification checklist MUST be read-only and MUST NOT trigger any external calls (e.g., no provider API calls, no HTTP calls, no background jobs started as a side effect of rendering). + +- **FR-005 — Start verification creates a run**: Starting verification MUST create (or reuse, per dedupe policy) a new verification run record and begin executing the verification checks using existing background processing. + +- **FR-006 — Dedupe / idempotency**: If a verification run is already active for the same identity (tenant + provider connection + flow), the system MUST NOT start a duplicate active run; it MUST present a clear “already running” outcome and an affordance to view the active run/report. + + If no run is active (including when the most recent run is completed or failed), “Start verification” MUST create a new run. + +- **FR-007 — Capability-first authorization**: Permission checks for viewing and starting verification MUST reference the canonical capability registry (no string-literal capability checks in feature code). + +- **FR-008 — RBAC UX semantics**: Non-members attempting to access tenant-scoped verification pages/routes MUST receive not-found responses. Members lacking the “start” capability MUST be able to view reports but MUST NOT be able to start verification (UI disabled + server-side enforcement). + + Viewing reports MUST NOT require the start capability. + +- **FR-009 — Standardized UI semantics**: The viewer MUST render consistent status labels, a summary banner (e.g., Ready / Needs attention / Blocked), and per-check expandable details with standardized “Next steps” calls-to-action. + +- **FR-010 — Reuse across suite**: The framework MUST be adoptable by multiple verification flows without re-implementing viewer logic, including: managed tenant onboarding verification, provider connection verification, RBAC setup verification, consent & permission verification, and future readiness/health checks. + +- **FR-011 — Auditing**: Starting and completing verification MUST emit audit events with stable action identifiers and redaction rules, recording minimal metadata (workspace/tenant identifiers, run identifier, and result counts). + +### Key Entities *(include if feature involves data)* + +- **Verification Flow**: A named verification context (e.g., managed tenant onboarding) that defines which checks run. +- **Verification Identity (Scope)**: The set of identifiers that uniquely represent “what is being verified” (tenant + provider connection + flow). +- **Verification Run**: A single execution attempt for a given identity that produces a report (and is auditable). +- **Verification Report**: A versioned, structured document attached to a run, containing summary and check results. +- **Check Definition**: A reusable definition of an atomic readiness check (key, title, expected preconditions, severity, blocking behavior). +- **Check Result**: The outcome of executing a check within a report. +- **Reason Code**: A stable, machine-readable classification of why a check is pass/fail/warn/skip. +- **Next Step**: An actionable remediation hint (label + optional destination/action) that helps the operator resolve a failed check. +- **Evidence Pointer**: Safe references that support diagnostics (IDs, masked strings, hashes), without exposing secrets. + +### Assumptions + +- A run-tracking mechanism already exists and can store an attached, versioned verification report per run. +- A canonical capability registry exists and is the source of truth for permission checks. +- An audit logging mechanism exists that can record start/complete events with redaction. +- Verification execution uses existing background processing patterns (no new observability platform is introduced). + +### Dependencies + +- Workspace membership and tenant-scoped authorization boundaries are already modeled. +- Run visibility rules support “deny-as-not-found” behavior for non-members. +- UI surfaces exist (or can be added) where “Next steps” can route users for remediation. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001 (Clarity)**: In a usability test with a pre-seeded failed report, 90% of operators can identify the top blocking failure and the recommended next step within 60 seconds. +- **SC-002 (Determinism)**: When “Start verification” is triggered repeatedly for the same identity while a run is active, the system starts at most 1 active run (0 duplicates) and always provides a path to view the active run. +- **SC-003 (Safety / data minimization)**: Verification reports contain no secrets or tokens; evidence is limited to safe pointers (validated by automated tests and/or static checks). +- **SC-004 (Performance)**: The verification report viewer renders within 200ms server time for a typical report of up to 50 checks. +- **SC-005 (Authorization)**: Non-member access to tenant-scoped verification pages results in not-found responses in 100% of tested cases; members without the start capability cannot execute start actions in 100% of tested cases. diff --git a/specs/074-verification-checklist/tasks.md b/specs/074-verification-checklist/tasks.md new file mode 100644 index 0000000..35c6680 --- /dev/null +++ b/specs/074-verification-checklist/tasks.md @@ -0,0 +1,120 @@ +# Tasks: 074 Verification Checklist Framework + +**Input**: Design documents from `/specs/074-verification-checklist/` +**Prerequisites**: plan.md (required), spec.md (required for user stories), research.md, data-model.md, contracts/ + +**Tests**: Required (Pest). + +--- + +## Phase 1: Foundational (Blocking Prerequisites) + +- [x] T001 [US1] Add example reports under specs: `specs/074-verification-checklist/contracts/examples/*.json` (pass/fail/warn/running) aligned to `contracts/verification-report.schema.json`. +- [x] T002 [P] [US1] Add a small schema validation helper for reports (pure PHP, no external deps) in `app/Support/Verification/VerificationReportSchema.php` (version parsing + shape validation + graceful fallback). +- [x] T003 [P] [US1] Add report redaction/sanitization utility in `app/Support/Verification/VerificationReportSanitizer.php` (denylist keys/values; enforce evidence pointers only). +- [x] T004 [US1] Add value objects (or typed arrays) for report/check concepts in `app/Support/Verification/*` (status/severity enums or constants) to avoid ad-hoc strings throughout UI. + +**RBAC & UX prereqs** + +- [x] T005 [US3] Decide and document the start capability used per verification flow (v1: use `Capabilities::PROVIDER_RUN` for `provider.connection.check`; prefer existing constants in `app/Support/Auth/Capabilities.php`). +- [x] T006 [US3] Add/confirm central UI enforcement helper usage for “visible-but-disabled with tooltip” in verification start UI (use tenant-scoped `app/Support/Rbac/UiEnforcement.php` with a resolved `Tenant` record). + +**Badges (BADGE-001)** + +- [x] T007 [P] [US1] Add badge domains for verification status/severity in `app/Support/Badges/BadgeDomain.php`. +- [x] T008 [P] [US1] Add domain mappers in `app/Support/Badges/Domains/*` (e.g., `VerificationCheckStatusBadge`, `VerificationCheckSeverityBadge`). +- [x] T009 [US1] Register domains in `app/Support/Badges/BadgeCatalog.php`. +- [x] T010 [US1] Add mapping tests for new badge domains in `tests/Unit/Badges/*`. + +**Checkpoint**: Report contract + sanitizer + badge domains exist; UI work can start. + +--- + +## Phase 2: User Story 1 — Operator sees what’s wrong (Priority: P1) + +**Goal**: Render a structured, DB-only verification report viewer for a run. + +**Independent Test**: Seed an `OperationRun` with `context.verification_report` and assert the viewer renders the correct summary + per-check details, with no outbound HTTP. + +### Tests (write first) + +- [x] T011 [US1] Add a viewer DB-only test (no outbound HTTP, no job dispatch) in `tests/Feature/Verification/VerificationReportViewerDbOnlyTest.php` using `Http::fake()` + `Bus::fake()` and asserting no requests / no dispatch during page render (including a second render to cover Livewire refresh/poll paths). +- [x] T012 [US1] Add a redaction test in `tests/Feature/Verification/VerificationReportRedactionTest.php` to ensure forbidden keys/values never appear in stored/rendered evidence. +- [x] T013 [US1] Add a “malformed/missing report” viewer test in `tests/Feature/Verification/VerificationReportMissingOrMalformedTest.php` (safe empty state). + +### Implementation + +- [x] T014 [US1] Create a reusable viewer Blade partial in `resources/views/filament/components/verification-report-viewer.blade.php` (summary banner + counts + collapsible checks + next steps links-only). +- [x] T015 [US1] Create a Filament view entry/helper to render the viewer from an `OperationRun` in `app/Filament/Support/VerificationReportViewer.php` (or existing Filament helpers location), using only DB values. +- [x] T016 [US1] Integrate viewer into Monitoring → Operations run view page: update `app/Filament/Resources/OperationRunResource.php` (infolist) to show the verification report section when `context.verification_report` exists. + +**Checkpoint**: A seeded report is readable in Monitoring; viewer is DB-only. + +--- + +## Phase 3: User Story 2 — Deterministic starts (Priority: P2) + +**Goal**: Starting verification is idempotent while active (dedupe) and guides users to the active run. + +**Independent Test**: Start verification twice for the same identity and assert a single active run is used. + +### Tests + +- [x] T017 [US2] Add a dedupe regression test in `tests/Feature/Verification/VerificationStartDedupeTest.php` asserting repeated starts reuse the same active run (leveraging the existing `OperationRunService::ensureRunWithIdentity()` behavior). +- [x] T018 [US2] Add a “new run after completion” test in `tests/Feature/Verification/VerificationStartAfterCompletionTest.php`. + +### Implementation + +- [x] T019 [US2] Add (or adapt) a small “start verification” service wrapper in `app/Services/Verification/StartVerification.php` that: authorizes, creates/reuses a run identity, enqueues a verifier job, and returns the run. +- [x] T020 [US2] Update the managed tenant onboarding verification step to route through the shared starter and replace the binary status UI with the shared verification report viewer (or a safe empty state) in `app/Filament/Pages/Workspaces/ManagedTenantOnboardingWizard.php`. +- [x] T021 [US2] Update provider connection verification start surface(s) (where present) to route through the same shared starter. + +**Checkpoint**: Starts are deterministic and route users to the active run. + +--- + +## Phase 4: User Story 3 — Least-privilege and safe disclosure (Priority: P3) + +**Goal**: View vs start capability split; non-members get 404; members lacking start capability get 403 on execution. + +**Independent Test**: Readonly member can view report but cannot start; non-member cannot discover tenant/run. + +### Tests + +- [x] T022 [US3] Add authorization tests for view vs start in `tests/Feature/Verification/VerificationAuthorizationTest.php` covering: + - tenant non-member → 404 on view + start + - tenant member without start capability → can view, start returns forbidden (403) + - tenant member with start capability → can start + +### Implementation + +- [x] T023 [US3] Ensure start actions enforce server-side authorization via Gate/Policy (no UI-only enforcement) and use capability constants from `app/Support/Auth/Capabilities.php`. +- [x] T024 [US3] Ensure tenant-scope non-membership yields deny-as-not-found behavior for verification routes/actions (align with existing tenant routing patterns and helpers). + +**Checkpoint**: Authorization behavior matches RBAC-UX contract. + +--- + +## Phase 5: Audit & Completion Events (Cross-cutting) + +- [x] T025 [US1] Add a stable audit action ID for verification completion in `app/Support/Audit/AuditActionId.php`. +- [x] T026 [US1] Emit a completion audit event when a verification run finalizes (where run completion is set) using `app/Services/Audit/WorkspaceAuditLogger.php` with redacted metadata (run id + counts only). +- [x] T030 [US1] Add a report writer in `app/Support/Verification/VerificationReportWriter.php` that builds `context.verification_report`, derives `summary.overall` deterministically, enforces reason codes + evidence pointer-only policy, and runs sanitizer before persistence. +- [x] T031 [US1] Integrate the report writer into `app/Jobs/ProviderConnectionHealthCheckJob.php` so `provider.connection.check` writes a compliant `verification_report` to the run (both success and failure paths) before marking the run completed. +- [x] T032 [US1] Add a report-writing integration test in `tests/Feature/Verification/ProviderConnectionHealthCheckWritesReportTest.php` ensuring the run ends with a valid, sanitized `context.verification_report` (and no forbidden evidence fields). + +--- + +## Phase 6: Polish & Regression Guards + +- [x] T027 [P] Add UI polish for empty/missing report state in the viewer (no leaks of internal details). +- [x] T028 Run formatting: `vendor/bin/sail bin pint --dirty`. +- [x] T029 Run targeted tests: `vendor/bin/sail artisan test --compact tests/Feature/Verification`. + +--- + +## Dependencies & Execution Order + +- Phase 1 (Foundational) blocks all other phases. +- US1 can start after Phase 1; US2/US3 can proceed after Phase 1 but should reuse US1 primitives (viewer + sanitizer + badges). +- Audit completion (Phase 5) depends on the shared verification job/service that finalizes runs. diff --git a/tests/Feature/Verification/ProviderConnectionHealthCheckWritesReportTest.php b/tests/Feature/Verification/ProviderConnectionHealthCheckWritesReportTest.php new file mode 100644 index 0000000..f99326c --- /dev/null +++ b/tests/Feature/Verification/ProviderConnectionHealthCheckWritesReportTest.php @@ -0,0 +1,165 @@ +create([ + 'tenant_id' => (int) $tenant->getKey(), + 'provider' => 'microsoft', + 'entra_tenant_id' => fake()->uuid(), + ]); + + ProviderCredential::factory()->create([ + 'provider_connection_id' => (int) $connection->getKey(), + 'payload' => [ + 'tenant_id' => (string) $connection->entra_tenant_id, + 'client_id' => fake()->uuid(), + 'client_secret' => fake()->sha1(), + ], + ]); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'running', + 'outcome' => 'pending', + 'context' => [ + 'provider_connection_id' => (int) $connection->getKey(), + ], + ]); + + $this->mock(GraphClientInterface::class, function ($mock): void { + $mock->shouldReceive('getOrganization') + ->once() + ->andReturn(new GraphResponse(false, [], 401, ['Bearer super-secret-token'])); + }); + + $job = new ProviderConnectionHealthCheckJob( + tenantId: (int) $tenant->getKey(), + userId: (int) $user->getKey(), + providerConnectionId: (int) $connection->getKey(), + operationRun: $run, + ); + + $job->handle( + healthCheck: app(MicrosoftProviderHealthCheck::class), + runs: app(OperationRunService::class), + ); + + $run = $run->fresh(); + + expect($run)->not->toBeNull(); + expect($run->status)->toBe('completed'); + expect($run->outcome)->toBe('failed'); + + $context = is_array($run->context) ? $run->context : []; + $report = $context['verification_report'] ?? null; + + expect($report)->toBeArray(); + expect(VerificationReportSchema::isValidReport($report))->toBeTrue(); + expect(json_encode($report))->not->toContain('Bearer '); + expect($report['checks'][0]['reason_code'] ?? null)->toBe('authentication_failed'); + + foreach (($report['checks'] ?? []) as $check) { + expect($check)->toBeArray(); + + foreach (($check['evidence'] ?? []) as $pointer) { + expect($pointer)->toBeArray(); + expect(array_keys($pointer))->toEqualCanonicalizing(['kind', 'value']); + } + } + + $audit = AuditLog::query() + ->where('workspace_id', (int) $tenant->workspace_id) + ->where('action', AuditActionId::VerificationCompleted->value) + ->latest('id') + ->first(); + + expect($audit)->not->toBeNull(); + expect($audit?->metadata)->toMatchArray([ + 'operation_run_id' => (int) $run->getKey(), + ]); +}); + +it('writes a verification report for successful provider connection checks', function (): void { + [$user, $tenant] = createUserWithTenant(role: 'operator'); + + $connection = ProviderConnection::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'provider' => 'microsoft', + 'entra_tenant_id' => fake()->uuid(), + ]); + + ProviderCredential::factory()->create([ + 'provider_connection_id' => (int) $connection->getKey(), + 'payload' => [ + 'tenant_id' => (string) $connection->entra_tenant_id, + 'client_id' => fake()->uuid(), + 'client_secret' => fake()->sha1(), + ], + ]); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'running', + 'outcome' => 'pending', + 'context' => [ + 'provider_connection_id' => (int) $connection->getKey(), + ], + ]); + + $this->mock(GraphClientInterface::class, function ($mock): void { + $mock->shouldReceive('getOrganization') + ->once() + ->andReturn(new GraphResponse(true, [ + 'id' => 'org_123', + 'displayName' => 'Org 123', + ], 200)); + }); + + $job = new ProviderConnectionHealthCheckJob( + tenantId: (int) $tenant->getKey(), + userId: (int) $user->getKey(), + providerConnectionId: (int) $connection->getKey(), + operationRun: $run, + ); + + $job->handle( + healthCheck: app(MicrosoftProviderHealthCheck::class), + runs: app(OperationRunService::class), + ); + + $run = $run->fresh(); + + expect($run)->not->toBeNull(); + expect($run->status)->toBe('completed'); + expect($run->outcome)->toBe('succeeded'); + + $context = is_array($run->context) ? $run->context : []; + $report = $context['verification_report'] ?? null; + + expect($report)->toBeArray(); + expect(VerificationReportSchema::isValidReport($report))->toBeTrue(); + expect($report['summary']['counts'] ?? [])->toMatchArray([ + 'total' => 1, + 'pass' => 1, + 'fail' => 0, + ]); +}); diff --git a/tests/Feature/Verification/VerificationAuthorizationTest.php b/tests/Feature/Verification/VerificationAuthorizationTest.php new file mode 100644 index 0000000..4086fd0 --- /dev/null +++ b/tests/Feature/Verification/VerificationAuthorizationTest.php @@ -0,0 +1,112 @@ +create(); + $otherTenant = Tenant::factory()->create(); + + [$user] = createUserWithTenant($otherTenant, role: 'readonly'); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'completed', + 'outcome' => 'failed', + 'context' => [ + 'verification_report' => json_decode( + (string) file_get_contents(base_path('specs/074-verification-checklist/contracts/examples/fail.json')), + true, + 512, + JSON_THROW_ON_ERROR, + ), + ], + ]); + + $this->actingAs($user) + ->get(OperationRunResource::getUrl('view', ['record' => $run], tenant: $tenant)) + ->assertStatus(404); + + $connection = ProviderConnection::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + ]); + + expect(fn () => app(StartVerification::class)->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + ))->toThrow(NotFoundHttpException::class); +}); + +it('allows readonly members to view verification reports but forbids starting verification', function (): void { + Queue::fake(); + + [$user, $tenant] = createUserWithTenant(role: 'readonly'); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'completed', + 'outcome' => 'failed', + 'context' => [ + 'verification_report' => json_decode( + (string) file_get_contents(base_path('specs/074-verification-checklist/contracts/examples/fail.json')), + true, + 512, + JSON_THROW_ON_ERROR, + ), + ], + ]); + + $this->actingAs($user) + ->get(OperationRunResource::getUrl('view', ['record' => $run], tenant: $tenant)) + ->assertOk() + ->assertSee('Verification report'); + + $connection = ProviderConnection::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + ]); + + expect(fn () => app(StartVerification::class)->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + ))->toThrow(AuthorizationException::class); +}); + +it('allows members with start capability to start verification', function (): void { + Queue::fake(); + + [$user, $tenant] = createUserWithTenant(role: 'operator'); + + $connection = ProviderConnection::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'provider' => 'microsoft', + 'entra_tenant_id' => fake()->uuid(), + ]); + + $result = app(StartVerification::class)->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + ); + + expect($result->status)->toBe('started'); + expect($result->run->type)->toBe('provider.connection.check'); + expect($result->run->tenant_id)->toBe((int) $tenant->getKey()); + expect($result->run->context)->toMatchArray([ + 'provider_connection_id' => (int) $connection->getKey(), + ]); +}); diff --git a/tests/Feature/Verification/VerificationReportMissingOrMalformedTest.php b/tests/Feature/Verification/VerificationReportMissingOrMalformedTest.php new file mode 100644 index 0000000..b28a756 --- /dev/null +++ b/tests/Feature/Verification/VerificationReportMissingOrMalformedTest.php @@ -0,0 +1,59 @@ +actingAs($user); + + $tenant->makeCurrent(); + Filament::setTenant($tenant, true); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'completed', + 'outcome' => 'failed', + 'context' => [], + ]); + + assertNoOutboundHttp(function () use ($run): void { + Livewire::test(ViewOperationRun::class, ['record' => $run->getRouteKey()]) + ->assertSee('Verification report') + ->assertSee('Verification report unavailable'); + }); +}); + +it('shows a safe empty state when a verification report is malformed', function (): void { + [$user, $tenant] = createUserWithTenant(role: 'operator'); + $this->actingAs($user); + + $tenant->makeCurrent(); + Filament::setTenant($tenant, true); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'completed', + 'outcome' => 'failed', + 'context' => [ + 'verification_report' => [ + 'schema_version' => '1.0.0', + 'flow' => 'provider.connection.check', + ], + ], + ]); + + assertNoOutboundHttp(function () use ($run): void { + Livewire::test(ViewOperationRun::class, ['record' => $run->getRouteKey()]) + ->assertSee('Verification report') + ->assertSee('Verification report unavailable'); + }); +}); diff --git a/tests/Feature/Verification/VerificationReportRedactionTest.php b/tests/Feature/Verification/VerificationReportRedactionTest.php new file mode 100644 index 0000000..38b192d --- /dev/null +++ b/tests/Feature/Verification/VerificationReportRedactionTest.php @@ -0,0 +1,47 @@ +actingAs($user); + + $tenant->makeCurrent(); + Filament::setTenant($tenant, true); + + $report = json_decode( + (string) file_get_contents(base_path('specs/074-verification-checklist/contracts/examples/fail.json')), + true, + 512, + JSON_THROW_ON_ERROR, + ); + + $report['checks'][0]['evidence'][] = ['kind' => 'authorization', 'value' => 'Bearer abc.def.ghi']; + $report['checks'][0]['evidence'][] = ['kind' => 'access_token', 'value' => 'super-secret']; + $report['checks'][0]['message'] = 'Authorization: Bearer abc.def.ghi access_token=super-secret'; + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'completed', + 'outcome' => 'failed', + 'context' => [ + 'verification_report' => $report, + ], + ]); + + assertNoOutboundHttp(function () use ($run): void { + Livewire::test(ViewOperationRun::class, ['record' => $run->getRouteKey()]) + ->assertSee('Verification report') + ->assertSee('Token acquisition works') + ->assertDontSee('access_token') + ->assertDontSee('Bearer abc.def.ghi') + ->assertDontSee('super-secret'); + }); +}); diff --git a/tests/Feature/Verification/VerificationReportViewerDbOnlyTest.php b/tests/Feature/Verification/VerificationReportViewerDbOnlyTest.php new file mode 100644 index 0000000..dd09b72 --- /dev/null +++ b/tests/Feature/Verification/VerificationReportViewerDbOnlyTest.php @@ -0,0 +1,50 @@ +actingAs($user); + + $tenant->makeCurrent(); + Filament::setTenant($tenant, true); + + $report = json_decode( + (string) file_get_contents(base_path('specs/074-verification-checklist/contracts/examples/fail.json')), + true, + 512, + JSON_THROW_ON_ERROR, + ); + + $run = OperationRun::factory()->create([ + 'tenant_id' => (int) $tenant->getKey(), + 'user_id' => (int) $user->getKey(), + 'type' => 'provider.connection.check', + 'status' => 'completed', + 'outcome' => 'failed', + 'context' => [ + 'verification_report' => $report, + ], + ]); + + assertNoOutboundHttp(function () use ($run): void { + $component = Livewire::test(ViewOperationRun::class, ['record' => $run->getRouteKey()]) + ->assertSee('Verification report') + ->assertSee('Blocked') + ->assertSee('Token acquisition works'); + + $component + ->call('$refresh') + ->assertSee('Token acquisition works'); + }); + + Bus::assertNothingDispatched(); +}); diff --git a/tests/Feature/Verification/VerificationStartAfterCompletionTest.php b/tests/Feature/Verification/VerificationStartAfterCompletionTest.php new file mode 100644 index 0000000..e36a5b6 --- /dev/null +++ b/tests/Feature/Verification/VerificationStartAfterCompletionTest.php @@ -0,0 +1,62 @@ +actingAs($user); + + $tenant->makeCurrent(); + Filament::setTenant($tenant, true); + + $connection = ProviderConnection::factory()->create([ + 'tenant_id' => $tenant->getKey(), + 'provider' => 'microsoft', + 'entra_tenant_id' => fake()->uuid(), + ]); + + $starter = app(StartVerification::class); + + $first = $starter->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + ); + + /** @var OperationRun $firstRun */ + $firstRun = $first->run->refresh(); + + app(OperationRunService::class)->updateRun( + $firstRun, + status: OperationRunStatus::Completed->value, + outcome: OperationRunOutcome::Succeeded->value, + ); + + $second = $starter->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + ); + + expect($second->status)->toBe('started'); + expect($second->run->getKey())->not->toBe($firstRun->getKey()); + + expect(OperationRun::query() + ->where('tenant_id', $tenant->getKey()) + ->where('type', 'provider.connection.check') + ->count())->toBe(2); + + Queue::assertPushed(ProviderConnectionHealthCheckJob::class, 2); +}); diff --git a/tests/Feature/Verification/VerificationStartDedupeTest.php b/tests/Feature/Verification/VerificationStartDedupeTest.php new file mode 100644 index 0000000..835cda8 --- /dev/null +++ b/tests/Feature/Verification/VerificationStartDedupeTest.php @@ -0,0 +1,53 @@ +actingAs($user); + + $tenant->makeCurrent(); + Filament::setTenant($tenant, true); + + $connection = ProviderConnection::factory()->create([ + 'tenant_id' => $tenant->getKey(), + 'provider' => 'microsoft', + 'entra_tenant_id' => fake()->uuid(), + ]); + + $starter = app(StartVerification::class); + + $first = $starter->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + extraContext: ['wizard' => ['flow' => 'managed_tenant_onboarding']], + ); + + $second = $starter->providerConnectionCheck( + tenant: $tenant, + connection: $connection, + initiator: $user, + extraContext: ['wizard' => ['flow' => 'managed_tenant_onboarding']], + ); + + expect($first->run->getKey())->toBe($second->run->getKey()); + expect($first->status)->toBe('started'); + expect($second->status)->toBe('deduped'); + + expect(OperationRun::query() + ->where('tenant_id', $tenant->getKey()) + ->where('type', 'provider.connection.check') + ->count())->toBe(1); + + Queue::assertPushed(ProviderConnectionHealthCheckJob::class, 1); +}); diff --git a/tests/Unit/Badges/VerificationBadgesTest.php b/tests/Unit/Badges/VerificationBadgesTest.php new file mode 100644 index 0000000..5e2f0f8 --- /dev/null +++ b/tests/Unit/Badges/VerificationBadgesTest.php @@ -0,0 +1,68 @@ +label)->toBe('Pass'); + expect($pass->color)->toBe('success'); + + $fail = BadgeCatalog::spec(BadgeDomain::VerificationCheckStatus, 'fail'); + expect($fail->label)->toBe('Fail'); + expect($fail->color)->toBe('danger'); + + $warn = BadgeCatalog::spec(BadgeDomain::VerificationCheckStatus, 'warn'); + expect($warn->label)->toBe('Warn'); + expect($warn->color)->toBe('warning'); + + $skip = BadgeCatalog::spec(BadgeDomain::VerificationCheckStatus, 'skip'); + expect($skip->label)->toBe('Skipped'); + expect($skip->color)->toBe('gray'); + + $running = BadgeCatalog::spec(BadgeDomain::VerificationCheckStatus, 'running'); + expect($running->label)->toBe('Running'); + expect($running->color)->toBe('info'); +}); + +it('maps verification check severity values to canonical badge semantics', function (): void { + $info = BadgeCatalog::spec(BadgeDomain::VerificationCheckSeverity, 'info'); + expect($info->label)->toBe('Info'); + expect($info->color)->toBe('gray'); + + $low = BadgeCatalog::spec(BadgeDomain::VerificationCheckSeverity, 'low'); + expect($low->label)->toBe('Low'); + expect($low->color)->toBe('info'); + + $medium = BadgeCatalog::spec(BadgeDomain::VerificationCheckSeverity, 'medium'); + expect($medium->label)->toBe('Medium'); + expect($medium->color)->toBe('warning'); + + $high = BadgeCatalog::spec(BadgeDomain::VerificationCheckSeverity, 'high'); + expect($high->label)->toBe('High'); + expect($high->color)->toBe('danger'); + + $critical = BadgeCatalog::spec(BadgeDomain::VerificationCheckSeverity, 'critical'); + expect($critical->label)->toBe('Critical'); + expect($critical->color)->toBe('danger'); +}); + +it('maps verification report overall values to canonical badge semantics', function (): void { + $ready = BadgeCatalog::spec(BadgeDomain::VerificationReportOverall, 'ready'); + expect($ready->label)->toBe('Ready'); + expect($ready->color)->toBe('success'); + + $needsAttention = BadgeCatalog::spec(BadgeDomain::VerificationReportOverall, 'needs_attention'); + expect($needsAttention->label)->toBe('Needs attention'); + expect($needsAttention->color)->toBe('warning'); + + $blocked = BadgeCatalog::spec(BadgeDomain::VerificationReportOverall, 'blocked'); + expect($blocked->label)->toBe('Blocked'); + expect($blocked->color)->toBe('danger'); + + $running = BadgeCatalog::spec(BadgeDomain::VerificationReportOverall, 'running'); + expect($running->label)->toBe('Running'); + expect($running->color)->toBe('info'); +});