From 439a3b4edaa9bfa7fbbbe09a3814e30ce6b20622 Mon Sep 17 00:00:00 2001 From: Ahmed Darrazi Date: Tue, 23 Jun 2026 23:36:12 +0200 Subject: [PATCH] feat: harden json to jsonb data layer for trust payloads --- apps/platform/app/Models/BackupItem.php | 6 +- ...rt_trust_payload_json_columns_to_jsonb.php | 107 ++++ .../Database/JsonbDataLayerHardeningTest.php | 591 ++++++++++++++++++ .../checklists/requirements.md | 45 ++ .../implementation-report.md | 331 ++++++++++ .../plan.md | 401 ++++++++++++ .../spec.md | 441 +++++++++++++ .../tasks.md | 157 +++++ 8 files changed, 2078 insertions(+), 1 deletion(-) create mode 100644 apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php create mode 100644 apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php create mode 100644 specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md create mode 100644 specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md create mode 100644 specs/405-json-to-jsonb-data-layer-hardening/plan.md create mode 100644 specs/405-json-to-jsonb-data-layer-hardening/spec.md create mode 100644 specs/405-json-to-jsonb-data-layer-hardening/tasks.md diff --git a/apps/platform/app/Models/BackupItem.php b/apps/platform/app/Models/BackupItem.php index c75e7547..a1cd107e 100644 --- a/apps/platform/app/Models/BackupItem.php +++ b/apps/platform/app/Models/BackupItem.php @@ -174,7 +174,11 @@ public function resolvedDisplayName(): string // Scopes public function scopeWithAssignments($query) { + $arrayLengthExpression = $query->getConnection()->getDriverName() === 'pgsql' + ? 'jsonb_array_length(assignments::jsonb)' + : 'json_array_length(assignments)'; + return $query->whereNotNull('assignments') - ->whereRaw('json_array_length(assignments) > 0'); + ->whereRaw($arrayLengthExpression.' > 0'); } } diff --git a/apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php b/apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php new file mode 100644 index 00000000..c969e07e --- /dev/null +++ b/apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php @@ -0,0 +1,107 @@ + + */ + private const CONVERTED_JSON_COLUMNS = [ + ['alert_deliveries', 'payload'], + ['alert_rules', 'tenant_allowlist'], + ['audit_logs', 'metadata'], + ['backup_items', 'assignments'], + ['backup_items', 'metadata'], + ['backup_items', 'payload'], + ['backup_schedules', 'days_of_week'], + ['backup_schedules', 'policy_types'], + ['backup_sets', 'metadata'], + ['managed_environment_onboarding_sessions', 'state'], + ['managed_environment_permissions', 'details'], + ['managed_environments', 'metadata'], + ['managed_environments', 'rbac_canary_results'], + ['managed_environments', 'rbac_last_warnings'], + ['policies', 'metadata'], + ['policy_versions', 'assignments'], + ['policy_versions', 'metadata'], + ['policy_versions', 'scope_tags'], + ['policy_versions', 'secret_fingerprints'], + ['policy_versions', 'snapshot'], + ['restore_runs', 'group_mapping'], + ['restore_runs', 'metadata'], + ['restore_runs', 'preview'], + ['restore_runs', 'requested_items'], + ['restore_runs', 'results'], + ['tenant_settings', 'value'], + ['workspace_settings', 'value'], + ]; + + public function up(): void + { + if (DB::getDriverName() !== 'pgsql') { + return; + } + + foreach ($this->columnsGroupedByTable(self::CONVERTED_JSON_COLUMNS) as $table => $columns) { + $this->alterJsonColumnTypes($table, $columns, 'jsonb'); + } + } + + public function down(): void + { + if (DB::getDriverName() !== 'pgsql') { + return; + } + + foreach ($this->columnsGroupedByTable(array_reverse(self::CONVERTED_JSON_COLUMNS)) as $table => $columns) { + $this->alterJsonColumnTypes($table, $columns, 'json'); + } + } + + /** + * @param list $columns + * @return array> + */ + private function columnsGroupedByTable(array $columns): array + { + $grouped = []; + + foreach ($columns as [$table, $column]) { + $grouped[$table][] = $column; + } + + return $grouped; + } + + /** + * @param list $columns + */ + private function alterJsonColumnTypes(string $table, array $columns, string $type): void + { + $clauses = array_map( + fn (string $column): string => sprintf( + 'ALTER COLUMN %s TYPE %s USING %s::%s', + $this->quoteIdentifier($column), + $type, + $this->quoteIdentifier($column), + $type, + ), + $columns, + ); + + DB::statement(sprintf( + 'ALTER TABLE %s %s', + $this->quoteIdentifier($table), + implode(', ', $clauses), + )); + } + + private function quoteIdentifier(string $identifier): string + { + return '"'.str_replace('"', '""', $identifier).'"'; + } +}; diff --git a/apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php b/apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php new file mode 100644 index 00000000..55c8bb4f --- /dev/null +++ b/apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php @@ -0,0 +1,591 @@ +markTestSkipped('Spec 405 JSONB hardening is PostgreSQL-only.'); + } +}); + +it('converts every reviewed legacy json column to jsonb', function (): void { + $columnTypes = spec405ColumnTypes(); + + foreach (spec405ConvertedJsonColumns() as [$table, $column]) { + $qualifiedColumn = "{$table}.{$column}"; + + expect($columnTypes) + ->toHaveKey($qualifiedColumn) + ->and($columnTypes[$qualifiedColumn])->toBe('jsonb'); + } + + $remainingJsonColumns = DB::table('information_schema.columns') + ->where('table_schema', 'public') + ->where('data_type', 'json') + ->orderBy('table_name') + ->orderBy('column_name') + ->get(['table_name', 'column_name']) + ->map(fn (object $column): string => "{$column->table_name}.{$column->column_name}") + ->all(); + + expect($remainingJsonColumns)->toBeEmpty(); +}); + +it('does not introduce speculative gin indexes for converted payload columns', function (): void { + $convertedTables = collect(spec405ConvertedJsonColumns()) + ->pluck(0) + ->unique() + ->values() + ->all(); + + $convertedGinIndexes = DB::table('pg_indexes') + ->where('schemaname', 'public') + ->whereIn('tablename', $convertedTables) + ->where('indexdef', 'ilike', '%USING gin%') + ->orderBy('tablename') + ->orderBy('indexname') + ->get(['tablename', 'indexname']) + ->map(fn (object $index): string => "{$index->tablename}.{$index->indexname}") + ->all(); + + expect($convertedGinIndexes)->toBeEmpty(); +}); + +it('preserves existing json rows attributes constraints indexes and rollback query compatibility', function (): void { + $migration = spec405Migration(); + $columnContractsBefore = spec405ColumnContracts(); + $indexDefinitionsBefore = spec405IndexDefinitionsForConvertedTables(); + $constraintDefinitionsBefore = spec405ConstraintDefinitionsForConvertedTables(); + + $migration->down(); + + try { + $jsonColumnContracts = spec405ColumnContracts(); + + foreach (spec405ConvertedJsonColumns() as [$table, $column]) { + $qualifiedColumn = "{$table}.{$column}"; + + expect($jsonColumnContracts[$qualifiedColumn]['data_type'])->toBe('json') + ->and($jsonColumnContracts[$qualifiedColumn]['nullable'])->toBe($columnContractsBefore[$qualifiedColumn]['nullable']) + ->and($jsonColumnContracts[$qualifiedColumn]['default'])->toBe($columnContractsBefore[$qualifiedColumn]['default']); + } + + $ids = spec405CreateRepresentativePayloadRows(); + $payloadsBefore = spec405RepresentativePayloads($ids); + + expect(BackupItem::withAssignments()->pluck('id')->map(fn (int $id): int => $id)->all()) + ->toContain($ids['backup_item_id']); + + $migration->up(); + + expect(spec405ColumnContracts())->toBe($columnContractsBefore) + ->and(spec405IndexDefinitionsForConvertedTables())->toBe($indexDefinitionsBefore) + ->and(spec405ConstraintDefinitionsForConvertedTables())->toBe($constraintDefinitionsBefore) + ->and(spec405RepresentativePayloads($ids))->toMatchArray($payloadsBefore); + + expect(BackupItem::withAssignments()->pluck('id')->map(fn (int $id): int => $id)->all()) + ->toContain($ids['backup_item_id']); + + $migration->down(); + + expect(spec405ColumnTypes()['backup_items.assignments'])->toBe('json') + ->and(BackupItem::withAssignments()->pluck('id')->map(fn (int $id): int => $id)->all()) + ->toContain($ids['backup_item_id']); + } finally { + if ((spec405ColumnTypes()['backup_items.assignments'] ?? null) !== 'jsonb') { + $migration->up(); + } + } +}); + +it('preserves representative jsonb read write and cast behavior', function (): void { + $tenant = ManagedEnvironment::factory()->create([ + 'metadata' => ['source' => 'spec405', 'nested' => ['enabled' => true]], + ]); + + DB::table('managed_environments') + ->where('id', (int) $tenant->getKey()) + ->update([ + 'rbac_canary_results' => json_encode(['status' => 'pass', 'checks' => ['read' => true]], JSON_THROW_ON_ERROR), + 'rbac_last_warnings' => json_encode(['warnings' => ['scope review required']], JSON_THROW_ON_ERROR), + ]); + + $tenant->refresh(); + + expect($tenant->metadata)->toMatchArray(['nested' => ['enabled' => true], 'source' => 'spec405']) + ->and($tenant->rbac_canary_results)->toMatchArray(['checks' => ['read' => true], 'status' => 'pass']) + ->and($tenant->rbac_last_warnings)->toMatchArray(['warnings' => ['scope review required']]); + + $backupSet = BackupSet::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'metadata' => ['source' => 'spec405'], + ]); + $policy = Policy::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'metadata' => ['platform_family' => 'windows'], + ]); + + $backupItem = BackupItem::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'backup_set_id' => (int) $backupSet->getKey(), + 'policy_id' => (int) $policy->getKey(), + 'payload' => ['id' => 'policy-jsonb', 'settings' => [['name' => 'mode', 'value' => 'audit']]], + 'metadata' => ['scope_tag_names' => ['Default', 'Security']], + 'assignments' => [['id' => 'assignment-1', 'target' => ['groupId' => 'group-jsonb']]], + ]); + + expect($backupItem->refresh()->payload)->toMatchArray(['id' => 'policy-jsonb', 'settings' => [['name' => 'mode', 'value' => 'audit']]]) + ->and($backupItem->metadata)->toMatchArray(['scope_tag_names' => ['Default', 'Security']]) + ->and($backupItem->assignments)->toBe([['id' => 'assignment-1', 'target' => ['groupId' => 'group-jsonb']]]); + + $policyVersion = PolicyVersion::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'policy_id' => (int) $policy->getKey(), + 'snapshot' => ['displayName' => 'Spec 405 policy', 'settings' => [['id' => 'setting-1']]], + 'metadata' => ['source' => 'unit'], + 'assignments' => [['target' => ['groupId' => 'group-jsonb']]], + 'scope_tags' => ['0', 'security'], + 'secret_fingerprints' => ['snapshot' => [], 'assignments' => ['hash']], + ]); + + expect($policyVersion->refresh()->snapshot)->toMatchArray(['displayName' => 'Spec 405 policy', 'settings' => [['id' => 'setting-1']]]) + ->and($policyVersion->metadata)->toMatchArray(['source' => 'unit']) + ->and($policyVersion->assignments)->toBe([['target' => ['groupId' => 'group-jsonb']]]) + ->and($policyVersion->scope_tags)->toBe(['0', 'security']) + ->and($policyVersion->secret_fingerprints)->toMatchArray(['assignments' => ['hash'], 'snapshot' => []]); + + $restoreRun = RestoreRun::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'backup_set_id' => (int) $backupSet->getKey(), + 'requested_items' => [['policy_identifier' => 'policy-jsonb']], + 'preview' => [['action' => 'update', 'policy_identifier' => 'policy-jsonb']], + 'results' => ['items' => [['status' => 'applied']]], + 'metadata' => ['dry_run' => false], + 'group_mapping' => ['old-group' => 'new-group'], + ]); + + expect($restoreRun->refresh()->requested_items)->toBe([['policy_identifier' => 'policy-jsonb']]) + ->and($restoreRun->preview)->toBe([['action' => 'update', 'policy_identifier' => 'policy-jsonb']]) + ->and($restoreRun->results)->toMatchArray(['items' => [['status' => 'applied']]]) + ->and($restoreRun->metadata)->toMatchArray(['dry_run' => false]) + ->and($restoreRun->group_mapping)->toMatchArray(['old-group' => 'new-group']); + + $auditLog = AuditLog::query()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'action' => 'spec405.jsonb.regression', + 'resource_type' => 'policy', + 'resource_id' => 'policy-jsonb', + 'status' => 'success', + 'metadata' => ['_dedupe_key' => 'spec405-jsonb', 'reason' => 'schema regression'], + 'recorded_at' => now(), + ]); + + expect($auditLog->refresh()->metadata)->toHaveKey('_dedupe_key', 'spec405-jsonb'); + + $metadataLookupExists = AuditLog::query() + ->whereRaw("metadata ->> '_dedupe_key' = ?", ['spec405-jsonb']) + ->exists(); + + expect($metadataLookupExists)->toBeTrue(); + + $schedule = BackupSchedule::query()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'name' => 'Spec 405 weekly backup', + 'frequency' => 'weekly', + 'time_of_day' => '03:00:00', + 'days_of_week' => [1, 3, 5], + 'policy_types' => ['settingsCatalogPolicy', 'deviceConfiguration'], + ]); + + expect($schedule->refresh()->days_of_week)->toBe([1, 3, 5]) + ->and($schedule->policy_types)->toBe(['settingsCatalogPolicy', 'deviceConfiguration']); + + $alertRule = AlertRule::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'tenant_allowlist' => [(int) $tenant->getKey()], + ]); + $alertDelivery = AlertDelivery::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'alert_rule_id' => (int) $alertRule->getKey(), + 'payload' => ['title' => 'Spec 405', 'context' => ['severity' => 'high']], + ]); + $permission = ManagedEnvironmentPermission::query()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'permission_key' => 'DeviceManagementConfiguration.Read.All', + 'status' => 'granted', + 'details' => ['source' => 'graph', 'roles' => ['reader']], + 'last_checked_at' => now(), + ]); + $onboardingSession = ManagedEnvironmentOnboardingSession::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'state' => ['entra_tenant_id' => 'tenant-jsonb', 'step' => 'verify'], + ]); + $tenantSetting = TenantSetting::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'value' => ['keep_last' => 45], + ]); + $workspaceSetting = WorkspaceSetting::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'value' => ['keep_last' => 30], + ]); + + expect($backupSet->refresh()->metadata)->toMatchArray(['source' => 'spec405']) + ->and($policy->refresh()->metadata)->toMatchArray(['platform_family' => 'windows']) + ->and($alertRule->refresh()->tenant_allowlist)->toBe([(int) $tenant->getKey()]) + ->and($alertDelivery->refresh()->payload)->toMatchArray(['context' => ['severity' => 'high'], 'title' => 'Spec 405']) + ->and($permission->refresh()->details)->toMatchArray(['roles' => ['reader'], 'source' => 'graph']) + ->and($onboardingSession->refresh()->state)->toMatchArray(['entra_tenant_id' => 'tenant-jsonb']) + ->and($tenantSetting->refresh()->value)->toMatchArray(['keep_last' => 45]) + ->and($workspaceSetting->refresh()->value)->toMatchArray(['keep_last' => 30]); +}); + +it('keeps backup item assignment filtering compatible with jsonb columns', function (): void { + BackupItem::factory()->create(['assignments' => null]); + BackupItem::factory()->create(['assignments' => []]); + $withAssignments = BackupItem::factory()->create([ + 'assignments' => [ + ['id' => 'assignment-1', 'target' => ['groupId' => 'group-1']], + ], + ]); + + $resultIds = BackupItem::withAssignments() + ->pluck('id') + ->map(fn (int $id): int => (int) $id) + ->all(); + + expect($resultIds)->toBe([(int) $withAssignments->getKey()]); +}); + +/** + * @return list + */ +function spec405ConvertedJsonColumns(): array +{ + return [ + ['alert_deliveries', 'payload'], + ['alert_rules', 'tenant_allowlist'], + ['audit_logs', 'metadata'], + ['backup_items', 'assignments'], + ['backup_items', 'metadata'], + ['backup_items', 'payload'], + ['backup_schedules', 'days_of_week'], + ['backup_schedules', 'policy_types'], + ['backup_sets', 'metadata'], + ['managed_environment_onboarding_sessions', 'state'], + ['managed_environment_permissions', 'details'], + ['managed_environments', 'metadata'], + ['managed_environments', 'rbac_canary_results'], + ['managed_environments', 'rbac_last_warnings'], + ['policies', 'metadata'], + ['policy_versions', 'assignments'], + ['policy_versions', 'metadata'], + ['policy_versions', 'scope_tags'], + ['policy_versions', 'secret_fingerprints'], + ['policy_versions', 'snapshot'], + ['restore_runs', 'group_mapping'], + ['restore_runs', 'metadata'], + ['restore_runs', 'preview'], + ['restore_runs', 'requested_items'], + ['restore_runs', 'results'], + ['tenant_settings', 'value'], + ['workspace_settings', 'value'], + ]; +} + +function spec405Migration(): Migration +{ + return require database_path('migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php'); +} + +/** + * @return list + */ +function spec405ConvertedTables(): array +{ + return collect(spec405ConvertedJsonColumns()) + ->pluck(0) + ->unique() + ->values() + ->all(); +} + +/** + * @return array + */ +function spec405ColumnContracts(): array +{ + return DB::table('information_schema.columns') + ->where('table_schema', 'public') + ->whereIn('table_name', spec405ConvertedTables()) + ->orderBy('table_name') + ->orderBy('column_name') + ->get(['table_name', 'column_name', 'data_type', 'is_nullable', 'column_default']) + ->mapWithKeys(fn (object $column): array => [ + "{$column->table_name}.{$column->column_name}" => [ + 'data_type' => (string) $column->data_type, + 'nullable' => (string) $column->is_nullable, + 'default' => $column->column_default === null ? null : (string) $column->column_default, + ], + ]) + ->only(collect(spec405ConvertedJsonColumns())->map(fn (array $column): string => "{$column[0]}.{$column[1]}")->all()) + ->all(); +} + +/** + * @return array + */ +function spec405IndexDefinitionsForConvertedTables(): array +{ + return DB::table('pg_indexes') + ->where('schemaname', 'public') + ->whereIn('tablename', spec405ConvertedTables()) + ->orderBy('tablename') + ->orderBy('indexname') + ->get(['tablename', 'indexname', 'indexdef']) + ->mapWithKeys(fn (object $index): array => [ + "{$index->tablename}.{$index->indexname}" => (string) $index->indexdef, + ]) + ->all(); +} + +/** + * @return array + */ +function spec405ConstraintDefinitionsForConvertedTables(): array +{ + return DB::table('pg_constraint') + ->selectRaw('conrelid::regclass::text as table_name, conname, pg_get_constraintdef(oid) as definition') + ->whereRaw("connamespace = 'public'::regnamespace") + ->whereIn(DB::raw('conrelid::regclass::text'), spec405ConvertedTables()) + ->orderBy('table_name') + ->orderBy('conname') + ->get() + ->mapWithKeys(fn (object $constraint): array => [ + "{$constraint->table_name}.{$constraint->conname}" => (string) $constraint->definition, + ]) + ->all(); +} + +/** + * @return array + */ +function spec405CreateRepresentativePayloadRows(): array +{ + $tenant = ManagedEnvironment::factory()->create([ + 'metadata' => ['source' => 'spec405', 'nested' => ['enabled' => true]], + ]); + + DB::table('managed_environments') + ->where('id', (int) $tenant->getKey()) + ->update([ + 'rbac_canary_results' => json_encode(['status' => 'pass', 'checks' => ['read' => true]], JSON_THROW_ON_ERROR), + 'rbac_last_warnings' => json_encode(['warnings' => ['scope review required']], JSON_THROW_ON_ERROR), + ]); + + $backupSet = BackupSet::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'metadata' => ['source' => 'spec405'], + ]); + $policy = Policy::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'metadata' => ['platform_family' => 'windows'], + ]); + $backupItem = BackupItem::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'backup_set_id' => (int) $backupSet->getKey(), + 'policy_id' => (int) $policy->getKey(), + 'payload' => ['id' => 'policy-jsonb', 'settings' => [['name' => 'mode', 'value' => 'audit']]], + 'metadata' => ['scope_tag_names' => ['Default', 'Security']], + 'assignments' => [['id' => 'assignment-1', 'target' => ['groupId' => 'group-jsonb']]], + ]); + $policyVersion = PolicyVersion::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'policy_id' => (int) $policy->getKey(), + 'snapshot' => ['displayName' => 'Spec 405 policy', 'settings' => [['id' => 'setting-1']]], + 'metadata' => ['source' => 'unit'], + 'assignments' => [['target' => ['groupId' => 'group-jsonb']]], + 'scope_tags' => ['0', 'security'], + 'secret_fingerprints' => ['snapshot' => [], 'assignments' => ['hash']], + ]); + $restoreRun = RestoreRun::factory()->create([ + 'managed_environment_id' => (int) $tenant->getKey(), + 'workspace_id' => (int) $tenant->workspace_id, + 'backup_set_id' => (int) $backupSet->getKey(), + 'requested_items' => [['policy_identifier' => 'policy-jsonb']], + 'preview' => [['action' => 'update', 'policy_identifier' => 'policy-jsonb']], + 'results' => ['items' => [['status' => 'applied']]], + 'metadata' => ['dry_run' => false], + 'group_mapping' => ['old-group' => 'new-group'], + ]); + $auditLog = AuditLog::query()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'action' => 'spec405.jsonb.regression', + 'resource_type' => 'policy', + 'resource_id' => 'policy-jsonb', + 'status' => 'success', + 'metadata' => ['_dedupe_key' => 'spec405-jsonb', 'reason' => 'schema regression'], + 'recorded_at' => now(), + ]); + $schedule = BackupSchedule::query()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'name' => 'Spec 405 weekly backup', + 'frequency' => 'weekly', + 'time_of_day' => '03:00:00', + 'days_of_week' => [1, 3, 5], + 'policy_types' => ['settingsCatalogPolicy', 'deviceConfiguration'], + ]); + $alertRule = AlertRule::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'tenant_allowlist' => [(int) $tenant->getKey()], + ]); + $alertDelivery = AlertDelivery::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'alert_rule_id' => (int) $alertRule->getKey(), + 'payload' => ['title' => 'Spec 405', 'context' => ['severity' => 'high']], + ]); + $permission = ManagedEnvironmentPermission::query()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'permission_key' => 'DeviceManagementConfiguration.Read.All', + 'status' => 'granted', + 'details' => ['source' => 'graph', 'roles' => ['reader']], + 'last_checked_at' => now(), + ]); + $onboardingSession = ManagedEnvironmentOnboardingSession::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'state' => ['entra_tenant_id' => 'tenant-jsonb', 'step' => 'verify'], + ]); + $tenantSetting = TenantSetting::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'managed_environment_id' => (int) $tenant->getKey(), + 'value' => ['keep_last' => 45], + ]); + $workspaceSetting = WorkspaceSetting::factory()->create([ + 'workspace_id' => (int) $tenant->workspace_id, + 'value' => ['keep_last' => 30], + ]); + + return [ + 'alert_delivery_id' => (int) $alertDelivery->getKey(), + 'alert_rule_id' => (int) $alertRule->getKey(), + 'audit_log_id' => (int) $auditLog->getKey(), + 'backup_item_id' => (int) $backupItem->getKey(), + 'backup_schedule_id' => (int) $schedule->getKey(), + 'backup_set_id' => (int) $backupSet->getKey(), + 'managed_environment_id' => (int) $tenant->getKey(), + 'managed_environment_onboarding_session_id' => (int) $onboardingSession->getKey(), + 'managed_environment_permission_id' => (int) $permission->getKey(), + 'policy_id' => (int) $policy->getKey(), + 'policy_version_id' => (int) $policyVersion->getKey(), + 'restore_run_id' => (int) $restoreRun->getKey(), + 'tenant_setting_id' => (int) $tenantSetting->getKey(), + 'workspace_setting_id' => (int) $workspaceSetting->getKey(), + ]; +} + +/** + * @param array $ids + * @return array + */ +function spec405RepresentativePayloads(array $ids): array +{ + $alertDelivery = AlertDelivery::query()->findOrFail($ids['alert_delivery_id']); + $alertRule = AlertRule::query()->findOrFail($ids['alert_rule_id']); + $auditLog = AuditLog::query()->findOrFail($ids['audit_log_id']); + $backupItem = BackupItem::query()->findOrFail($ids['backup_item_id']); + $backupSchedule = BackupSchedule::query()->findOrFail($ids['backup_schedule_id']); + $backupSet = BackupSet::query()->findOrFail($ids['backup_set_id']); + $tenant = ManagedEnvironment::query()->findOrFail($ids['managed_environment_id']); + $onboardingSession = ManagedEnvironmentOnboardingSession::query()->findOrFail($ids['managed_environment_onboarding_session_id']); + $permission = ManagedEnvironmentPermission::query()->findOrFail($ids['managed_environment_permission_id']); + $policy = Policy::query()->findOrFail($ids['policy_id']); + $policyVersion = PolicyVersion::query()->findOrFail($ids['policy_version_id']); + $restoreRun = RestoreRun::query()->findOrFail($ids['restore_run_id']); + $tenantSetting = TenantSetting::query()->findOrFail($ids['tenant_setting_id']); + $workspaceSetting = WorkspaceSetting::query()->findOrFail($ids['workspace_setting_id']); + + return [ + 'alert_deliveries.payload' => $alertDelivery->payload, + 'alert_rules.tenant_allowlist' => $alertRule->tenant_allowlist, + 'audit_logs.metadata' => $auditLog->metadata, + 'backup_items.assignments' => $backupItem->assignments, + 'backup_items.metadata' => $backupItem->metadata, + 'backup_items.payload' => $backupItem->payload, + 'backup_schedules.days_of_week' => $backupSchedule->days_of_week, + 'backup_schedules.policy_types' => $backupSchedule->policy_types, + 'backup_sets.metadata' => $backupSet->metadata, + 'managed_environment_onboarding_sessions.state' => $onboardingSession->state, + 'managed_environment_permissions.details' => $permission->details, + 'managed_environments.metadata' => $tenant->metadata, + 'managed_environments.rbac_canary_results' => $tenant->rbac_canary_results, + 'managed_environments.rbac_last_warnings' => $tenant->rbac_last_warnings, + 'policies.metadata' => $policy->metadata, + 'policy_versions.assignments' => $policyVersion->assignments, + 'policy_versions.metadata' => $policyVersion->metadata, + 'policy_versions.scope_tags' => $policyVersion->scope_tags, + 'policy_versions.secret_fingerprints' => $policyVersion->secret_fingerprints, + 'policy_versions.snapshot' => $policyVersion->snapshot, + 'restore_runs.group_mapping' => $restoreRun->group_mapping, + 'restore_runs.metadata' => $restoreRun->metadata, + 'restore_runs.preview' => $restoreRun->preview, + 'restore_runs.requested_items' => $restoreRun->requested_items, + 'restore_runs.results' => $restoreRun->results, + 'tenant_settings.value' => $tenantSetting->value, + 'workspace_settings.value' => $workspaceSetting->value, + ]; +} + +/** + * @return array + */ +function spec405ColumnTypes(): array +{ + $tables = collect(spec405ConvertedJsonColumns()) + ->pluck(0) + ->unique() + ->values() + ->all(); + + return DB::table('information_schema.columns') + ->where('table_schema', 'public') + ->whereIn('table_name', $tables) + ->get(['table_name', 'column_name', 'data_type']) + ->mapWithKeys(fn (object $column): array => [ + "{$column->table_name}.{$column->column_name}" => (string) $column->data_type, + ]) + ->all(); +} diff --git a/specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md b/specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md new file mode 100644 index 00000000..cc5d44d3 --- /dev/null +++ b/specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md @@ -0,0 +1,45 @@ +# Specification Quality Checklist: Spec 405 - JSON-to-JSONB Data-layer Hardening + +**Purpose**: Validate specification completeness and quality before implementation preparation is handed off. +**Created**: 2026-06-23 +**Feature**: `specs/405-json-to-jsonb-data-layer-hardening/spec.md` + +## Content Quality + +- [x] Focused on product value, operator trust, and data-layer hardening risk. +- [x] No application implementation is performed by this preparation package. +- [x] Runtime implementation details are limited to expected migration/test/validation surfaces. +- [x] Mandatory Spec Kit sections are completed or explicitly marked N/A with rationale. +- [x] Scope excludes new product behavior, UI surfaces, authorization model changes, lifecycle semantics, and broad abstractions. + +## Requirement Completeness + +- [x] No unresolved clarification markers remain. +- [x] Requirements are testable and unambiguous. +- [x] Success criteria are measurable. +- [x] Acceptance scenarios are defined for inventory, conversion, and regression/reporting. +- [x] Edge cases are identified, including `jsonb` key order normalization and rollback limitations. +- [x] Dependencies and assumptions are identified. +- [x] Required final implementation report structure is defined. + +## Feature Readiness + +- [x] Functional requirements map to concrete tasks in `tasks.md`. +- [x] User stories cover the minimum viable implementation sequence. +- [x] The plan identifies likely affected repository surfaces without authorizing unrelated runtime edits. +- [x] Tasks include tests, PostgreSQL validation, focused browser proof, staging-like validation handling, and final close-out. + +## Constitution And Product Surface Readiness + +- [x] Spec Candidate Check is completed with approval class, score, and decision. +- [x] Completed-spec guardrail is explicit and related specs are read-only context. +- [x] No UI surface impact is checked with a clear rationale. +- [x] Product Surface Contract is handled as no-rendered-surface-change plus focused regression proof. +- [x] Browser verification is required as backend regression proof for existing payload-backed surfaces. +- [x] Human Product Sanity is scoped to unchanged trust semantics. +- [x] Proportionality review states no new runtime framework, persisted entity, status family, or UI taxonomy. +- [x] Test governance names PostgreSQL, feature, and focused browser lanes with fixture-cost controls. + +## Notes + +Preparation review result: pass. The package is ready for a separate implementation loop, provided the implementation agent completes the inventory matrix before writing migrations and preserves the no-product/no-UI boundary. diff --git a/specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md b/specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md new file mode 100644 index 00000000..6ca67811 --- /dev/null +++ b/specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md @@ -0,0 +1,331 @@ +# Spec 405 Implementation Report - JSON-to-JSONB Data-layer Hardening + +## 1. Candidate Gate Result + +**Current result**: `PASS WITH CONDITIONS`. + +Local PostgreSQL migration/type proof, rollback/down-up proof, schema-attribute proof, and focused model/query regression tests pass. The remaining condition is external staging/Dokploy validation; it is not accessible from this agent session, so the gate cannot honestly be stronger than `PASS WITH CONDITIONS`. + +## 2. Scope Confirmation + +In scope: + +- Inventory every live PostgreSQL `json` and `jsonb` column. +- Classify every live `json` column. +- Convert only reviewed `CONVERT` columns to `jsonb`. +- Preserve current payload semantics, casts, scoping, and query behavior. +- Add local PostgreSQL tests and focused regression proof. + +Out of scope and not changed: + +- UI surfaces, routes, navigation, Filament resources, panels, actions, forms, tables, widgets, or customer output. +- Authorization model, roles, capabilities, provider semantics, lifecycle semantics, normalized replacement tables, and new product concepts. +- Completed historical specs and their validation/task/smoke/browser history. +- Speculative JSONB indexes. + +Historical context reviewed as read-only: + +- Spec 400: no `implementation-report.md` present; spec/plan/tasks were treated as historical audit context only. +- Spec 401: backup confirmation and provider residual context reviewed; left unchanged. +- Spec 402: provider action residual closure context reviewed; left unchanged. +- Spec 403: evidence/currentness runtime closure context reviewed; left unchanged. +- Spec 404: management report PDF staging validation and `PASS WITH CONDITIONS` staging caveat reviewed; left unchanged. + +## 3. Dirty State + +Initial repository state: + +- Branch: `405-json-to-jsonb-data-layer-hardening` +- HEAD: `8918b357 feat: finish management report PDF staging validation (#475)` +- Initial dirty state: untracked `specs/405-json-to-jsonb-data-layer-hardening/` +- Session branch not created because the active spec package was already untracked in the working tree; work continued cautiously on the current feature branch. +- Initial `git diff --check`: passed. + +Final `git status --short --untracked-files=all`: + +```text + M apps/platform/app/Models/BackupItem.php +?? apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php +?? apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php +?? specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md +?? specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md +?? specs/405-json-to-jsonb-data-layer-hardening/plan.md +?? specs/405-json-to-jsonb-data-layer-hardening/spec.md +?? specs/405-json-to-jsonb-data-layer-hardening/tasks.md +``` + +The spec package was untracked at session start. This implementation added the migration, database test, implementation report, task completion updates, and the `BackupItem` runtime adjustment. + +## 4. JSON/JSONB Inventory Matrix + +Inventory source: PostgreSQL `information_schema.columns`, row/null counts from the live local database, 2026-06-23. Converted-column migration decision is direct `ALTER COLUMN ... TYPE jsonb USING ...::jsonb`, grouped per table when more than one column is converted on the same table. Local converted tables have small row counts and no JSON-column index rebuild requirement; staging must still validate actual lock duration and migration runtime before production promotion. + +| Column | Type | Rows | Nulls | Nullable | Default | Indexes | Classification | Decision | +|---|---:|---:|---:|---|---|---|---|---| +| alert_deliveries.payload | json | 0 | 0 | YES | none | none | CONVERT P2 | Alert delivery payload; direct conversion. | +| alert_rules.tenant_allowlist | json | 0 | 0 | YES | none | none | CONVERT P2 | Alert scoping list; direct conversion. | +| audit_logs.metadata | json | 1320 | 0 | YES | none | none | CONVERT P1 | Audit context queried by metadata key; direct conversion. | +| backup_items.payload | json | 246 | 0 | NO | none | none | CONVERT P1 | Critical backup snapshot payload; direct conversion. | +| backup_items.metadata | json | 246 | 0 | YES | none | none | CONVERT P1 | Backup metadata and warnings; direct conversion. | +| backup_items.assignments | json | 246 | 146 | YES | none | none | CONVERT P1 | Assignment proof payload; direct conversion and query function update. | +| backup_schedules.days_of_week | json | 0 | 0 | YES | none | none | CONVERT P2 | Schedule structured config; direct conversion. | +| backup_schedules.policy_types | json | 0 | 0 | NO | none | none | CONVERT P2 | Schedule policy-type filter; direct conversion. | +| backup_sets.metadata | json | 25 | 0 | YES | none | none | CONVERT P1 | Backup-set provenance metadata; direct conversion. | +| baseline_profiles.scope_jsonb | jsonb | 4 | 0 | NO | none | none | ALREADY_JSONB | No migration. | +| baseline_snapshot_items.meta_jsonb | jsonb | 102 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| baseline_snapshots.summary_jsonb | jsonb | 4 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| baseline_snapshots.completion_meta_jsonb | jsonb | 4 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| baseline_tenant_assignments.override_scope_jsonb | jsonb | 3 | 3 | YES | none | none | ALREADY_JSONB | No migration. | +| entra_groups.group_types | jsonb | 574 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| environment_review_sections.summary_payload | jsonb | 153 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| environment_review_sections.render_payload | jsonb | 153 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| environment_reviews.summary | jsonb | 28 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| evidence_snapshot_items.summary_payload | jsonb | 98 | 0 | NO | `{}` | `evidence_snapshot_items_payload_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| evidence_snapshots.summary | jsonb | 24 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| finding_exception_decisions.metadata | jsonb | 7 | 0 | NO | `{}` | `finding_exception_decisions_metadata_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| finding_exception_evidence_references.summary_payload | jsonb | 0 | 0 | NO | `{}` | `finding_exception_evidence_refs_payload_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| finding_exceptions.evidence_summary | jsonb | 9 | 0 | NO | `{}` | `finding_exceptions_evidence_summary_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| findings.evidence_jsonb | jsonb | 254 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| inventory_items.meta_jsonb | jsonb | 229 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| inventory_links.metadata | jsonb | 251 | 0 | YES | none | none | ALREADY_JSONB | No migration. | +| managed_environment_onboarding_sessions.state | json | 2 | 0 | YES | none | none | CONVERT P1 | Onboarding state payload; direct conversion. | +| managed_environment_permissions.details | json | 135 | 0 | YES | none | none | CONVERT P1 | Provider permission details; direct conversion. | +| managed_environment_triage_reviews.review_snapshot | jsonb | 0 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| managed_environments.metadata | json | 54 | 2 | YES | none | none | CONVERT P1 | Provider/environment metadata; direct conversion. | +| managed_environments.rbac_canary_results | json | 54 | 54 | YES | none | none | CONVERT P1 | RBAC readiness proof; direct conversion. | +| managed_environments.rbac_last_warnings | json | 54 | 54 | YES | none | none | CONVERT P1 | RBAC warning proof; direct conversion. | +| notifications.data | jsonb | 311 | 0 | NO | none | none | ALREADY_JSONB | No migration. | +| operation_runs.summary_counts | jsonb | 96 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| operation_runs.failure_summary | jsonb | 96 | 0 | NO | `[]` | none | ALREADY_JSONB | No migration. | +| operation_runs.context | jsonb | 96 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| platform_users.capabilities | jsonb | 1 | 0 | NO | `[]` | none | ALREADY_JSONB | No migration. | +| policies.metadata | json | 208 | 3 | YES | none | none | CONVERT P1 | Policy inventory metadata; direct conversion. | +| policy_versions.snapshot | json | 422 | 0 | NO | none | none | CONVERT P1 | Immutable policy snapshot; direct conversion. | +| policy_versions.metadata | json | 422 | 0 | YES | none | none | CONVERT P1 | Version metadata; direct conversion. | +| policy_versions.assignments | json | 422 | 216 | YES | none | none | CONVERT P1 | Version assignment snapshot; direct conversion. | +| policy_versions.scope_tags | json | 422 | 14 | YES | none | none | CONVERT P1 | Scope tag snapshot; direct conversion. | +| policy_versions.secret_fingerprints | json | 422 | 2 | YES | none | none | CONVERT P1 | Redaction integrity metadata; direct conversion. | +| product_usage_events.metadata | jsonb | 100 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| provider_connections.scopes_granted | jsonb | 16 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| provider_connections.metadata | jsonb | 16 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| restore_runs.requested_items | json | 4 | 2 | YES | none | none | CONVERT P1 | Restore request payload; direct conversion. | +| restore_runs.preview | json | 4 | 1 | YES | none | none | CONVERT P1 | Restore preview payload; direct conversion. | +| restore_runs.results | json | 4 | 0 | YES | none | none | CONVERT P1 | Restore execution result payload; direct conversion. | +| restore_runs.metadata | json | 4 | 0 | YES | none | none | CONVERT P1 | Restore metadata; direct conversion. | +| restore_runs.group_mapping | json | 4 | 4 | YES | none | none | CONVERT P1 | Restore group mapping; direct conversion. | +| review_packs.summary | jsonb | 23 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| review_packs.options | jsonb | 23 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| review_publication_resolution_cases.summary | jsonb | 3 | 0 | NO | `{}` | `review_publication_resolution_cases_summary_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| review_publication_resolution_cases.metadata | jsonb | 3 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| review_publication_resolution_steps.summary | jsonb | 17 | 0 | NO | `{}` | `review_publication_resolution_steps_summary_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| review_publication_resolution_steps.metadata | jsonb | 17 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| settings_catalog_definitions.raw | jsonb | 0 | 0 | NO | none | `idx_settings_catalog_definitions_raw_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| stored_reports.payload | jsonb | 35 | 0 | NO | none | `stored_reports_payload_gin` | ALREADY_JSONB | Existing justified GIN retained. | +| support_requests.context_envelope | jsonb | 0 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. | +| tenant_settings.value | json | 0 | 0 | NO | none | none | CONVERT P1 | Tenant-scoped setting value; direct conversion. | +| workspace_settings.value | json | 0 | 0 | NO | none | none | CONVERT P1 | Workspace-scoped setting value; direct conversion. | + +No live `json` column was left as `KEEP_JSON`, `DEPRECATED`, or `DECISION_REQUIRED`: every live `json` column had current model ownership, bounded row counts, existing cast/read semantics, and a trust-layer storage reason to align with the existing `jsonb` baseline. + +Inventory ownership, cast, and query continuation for converted columns: + +| Column | Model / owner | Existing cast or accessor | Existing query/rendered usage | Constraint/FK context | +|---|---|---|---|---| +| alert_deliveries.payload | `App\Models\AlertDelivery` | `array` | Alert delivery rendering/notification payload; no JSON predicate. | Workspace/environment/rule/destination FKs unchanged. | +| alert_rules.tenant_allowlist | `App\Models\AlertRule` | `array` | Alert tenant scoping list; no JSON predicate. | Workspace FK unchanged. | +| audit_logs.metadata | `App\Models\AuditLog` / `AuditRecorder` | `array` | `metadata ->> '_dedupe_key'`, `metadata->...` audit filters/tests. | Workspace/environment/operation FKs and audit scope check unchanged. | +| backup_items.payload | `App\Models\BackupItem` | `array` | Backup set detail, restore readiness, included item rendering. | Workspace/environment/backup/policy FKs unchanged. | +| backup_items.metadata | `App\Models\BackupItem` | `array` | Backup quality/source/warning helpers and rendered backup detail. | Workspace/environment/backup/policy FKs unchanged. | +| backup_items.assignments | `App\Models\BackupItem` | `array` | `scopeWithAssignments()`; assignment count/group helper rendering. | Workspace/environment/backup/policy FKs unchanged. | +| backup_schedules.days_of_week | `App\Models\BackupSchedule` | `array` | Schedule form/detail semantics; no JSON predicate. | Workspace/environment FKs and frequency check unchanged. | +| backup_schedules.policy_types | `App\Models\BackupSchedule` | `array` | Schedule policy-type selection; no JSON predicate. | Workspace/environment FKs and frequency check unchanged. | +| backup_sets.metadata | `App\Models\BackupSet` | `array` | Backup provenance/source helpers; `metadata->source` test path. | Workspace/environment FK unchanged. | +| managed_environment_onboarding_sessions.state | `App\Models\ManagedEnvironmentOnboardingSession` | `array` with allowed-key mutator | Onboarding resume/state rendering and historical migration reads. | Workspace/environment/user FKs and lifecycle constraints unchanged. | +| managed_environment_permissions.details | `App\Models\ManagedEnvironmentPermission` | `array` | Provider permission/readiness detail rendering; no JSON predicate. | Workspace/environment FK and permission unique key unchanged. | +| managed_environments.metadata | `App\Models\ManagedEnvironment` | `array` | Environment/provider metadata accessors and rendered environment context. | Workspace FK and tenant lifecycle constraints unchanged. | +| managed_environments.rbac_canary_results | `App\Models\ManagedEnvironment` | JSON decode accessor | RBAC readiness proof rendering. | Workspace FK and tenant lifecycle constraints unchanged. | +| managed_environments.rbac_last_warnings | `App\Models\ManagedEnvironment` | JSON decode accessor | RBAC warning rendering. | Workspace FK and tenant lifecycle constraints unchanged. | +| policies.metadata | `App\Models\Policy` | `array` | Policy inventory metadata rendering and helper access. | Workspace/environment FK unchanged. | +| policy_versions.snapshot | `App\Models\PolicyVersion` | `array` | Immutable version snapshot rendering/diff/backup provenance. | Workspace/environment/policy FKs unchanged. | +| policy_versions.metadata | `App\Models\PolicyVersion` | `array` | Version source/warning/integrity helpers. | Workspace/environment/policy FKs unchanged. | +| policy_versions.assignments | `App\Models\PolicyVersion` | `array` | Version assignment snapshot rendering; sibling hash index unchanged. | Workspace/environment/policy FKs unchanged. | +| policy_versions.scope_tags | `App\Models\PolicyVersion` | `array` | Scope tag snapshot rendering; sibling hash index unchanged. | Workspace/environment/policy FKs unchanged. | +| policy_versions.secret_fingerprints | `App\Models\PolicyVersion` | `array` | Redaction integrity helper. | Workspace/environment/policy FKs unchanged. | +| restore_runs.requested_items | `App\Models\RestoreRun` | `array` | Restore request/preview detail rendering. | Workspace/environment/backup FKs unchanged. | +| restore_runs.preview | `App\Models\RestoreRun` | `array` | Restore preview wizard/detail rendering. | Workspace/environment/backup FKs unchanged. | +| restore_runs.results | `App\Models\RestoreRun` | `array` | Restore result proof/detail rendering. | Workspace/environment/backup FKs unchanged. | +| restore_runs.metadata | `App\Models\RestoreRun` | `array` | Restore result/safety metadata and reconciliation helpers; no direct JSON predicate on this table. | Workspace/environment/backup FKs unchanged. | +| restore_runs.group_mapping | `App\Models\RestoreRun` | `array` | Restore group-mapping preview/detail rendering. | Workspace/environment/backup FKs unchanged. | +| tenant_settings.value | `App\Models\TenantSetting` | `array` | Tenant settings read/write; no JSON predicate. | Workspace/environment/user FKs unchanged. | +| workspace_settings.value | `App\Models\WorkspaceSetting` | `array` | Workspace settings read/write; no JSON predicate. | Workspace/user FKs unchanged. | + +Already-JSONB ownership groups were also reviewed and left unchanged: + +- Baseline/inventory/finding/evidence/review rows: existing `*_jsonb`, `summary`, `summary_payload`, `render_payload`, `meta_jsonb`, and evidence payload columns with current model/service ownership; existing GIN indexes retained where already present. +- OperationRun/provider/notification/support rows: existing `operation_runs.context`, summary/failure payloads, provider connection metadata/scopes, notification data, product usage metadata, and support request context already use `jsonb`. +- Stored report and review publication rows: `stored_reports.payload`, review pack summaries/options, and review publication resolution summaries/metadata already use `jsonb`; browser proof confirms rendered output remains stable. + +## 5. Migrations Added + +Added `apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php`. + +- Converts the 27 reviewed `CONVERT` columns with `ALTER TABLE ... ALTER COLUMN ... TYPE jsonb USING column::jsonb`. +- Groups multiple converted columns on the same table into a single `ALTER TABLE` statement to avoid repeated per-column table rewrites/lock windows where PostgreSQL can satisfy the changes together. +- Runs only when `DB::getDriverName() === 'pgsql'`. +- Preserves nullability, defaults, constraints, and existing non-JSON indexes by altering only the column type. +- Rollback converts the same columns back to `json` with `USING column::json`. +- Rollback limitation: `jsonb` canonicalizes object key ordering and duplicate JSON object keys. Semantic content is preserved, but byte-for-byte textual JSON representation is not guaranteed. +- Local rollback/down-up proof passed in the PostgreSQL lane, including `BackupItem::withAssignments()` while the column is temporarily `json`. + +## 6. Index Justification + +No new JSONB indexes were added. + +Usage scan found one conversion-affected raw JSON function path, `BackupItem::scopeWithAssignments()`, which filters by array length but does not justify a GIN or expression index at current row counts. Existing hash/sibling indexes and existing GIN indexes on already-JSONB tables remain unchanged. + +The new PostgreSQL test asserts no speculative GIN index exists on converted-column tables. + +## 7. Runtime Changes Made + +Changed `apps/platform/app/Models/BackupItem.php`: + +- `scopeWithAssignments()` now uses `jsonb_array_length(assignments::jsonb)` on PostgreSQL. +- The explicit cast keeps the scope compatible during pre-migration deploy windows and after a database-only rollback to `json`. +- Non-PostgreSQL test lanes keep `json_array_length(assignments)` to avoid breaking SQLite-style JSON test support. + +No model casts required changes. Laravel array casts continue to read/write `jsonb` columns correctly. + +## 8. Tests Added or Updated + +Added `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php` with PostgreSQL-only coverage: + +- All reviewed legacy `json` columns are `jsonb` after migrations. +- The 405 migration can run `down()` to `json` and back `up()` to `jsonb` while preserving representative existing rows. +- Nullability, defaults, table indexes, and constraints remain unchanged across the 405 down/up cycle. +- No remaining live public-schema `json` columns remain after the conversion migration. +- No speculative GIN index was introduced for converted columns. +- Representative read/write/cast behavior for alert, audit, backup, restore, policy, provider/environment, settings, and onboarding payload categories. +- Audit metadata key query using `metadata ->> '_dedupe_key'`. +- `BackupItem::withAssignments()` query path against both rollback-state `json` and migrated `jsonb`. + +Latest result: 5 tests passed, 260 assertions. + +Existing test run: + +- `apps/platform/tests/Unit/BackupItemTest.php` still passes after the driver-aware scope change. +- Latest result: 20 tests passed, 31 assertions. + +## 9. Data Validation + +Local PostgreSQL validation: + +- The migration/type test ran migrations on the PostgreSQL testing database and asserted every converted column is `jsonb`. +- The new down/up test temporarily rolled the 405 migration back to `json`, inserted representative rows across every converted column category, verified `BackupItem::withAssignments()` works in the rollback-state schema, migrated forward to `jsonb`, and compared decoded payload semantics. +- The same down/up test asserted nullability, defaults, table index definitions, and table constraint definitions remain unchanged after conversion. +- The same test asserted there are no remaining public-schema columns with `data_type = 'json'`. +- Representative non-sensitive samples were written and read back through Eloquent casts and direct JSONB key predicates. +- Row/null counts were recorded before conversion from the local development database; converted tables are small enough for direct `ALTER COLUMN` locally. Staging/Dokploy must still validate real runtime and lock behavior. + +Not performed in this session: + +- Production-sized timing. +- Staging/Dokploy migration runtime. +- Full byte-level dump comparison. This is not required because `jsonb` canonicalizes textual representation; semantic preservation was tested instead. + +## 10. Browser Proof + +Focused browser proof passed using existing payload-backed smoke tests. No UI files were changed; this proof is regression-only. + +Command: + +```bash +cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Browser/Spec403EvidenceCurrentnessRuntimeClosureSmokeTest.php tests/Browser/Spec394ProviderFreshnessPermissionSmokeTest.php tests/Browser/Spec371BackupSetProductizationSmokeTest.php tests/Browser/Spec335RestoreRunDetailProductizationSmokeTest.php tests/Browser/Spec379ManagementReportPdfSmokeTest.php +``` + +Result: + +```text +5 browser tests passed, 176 assertions, 19.87s +``` + +Covered surfaces: + +- Evidence Overview and OperationRun proof currentness: `Spec403EvidenceCurrentnessRuntimeClosureSmokeTest.php` +- Provider freshness and required-permission readiness: `Spec394ProviderFreshnessPermissionSmokeTest.php` +- Backup set list/detail decision hierarchy and included backup items: `Spec371BackupSetProductizationSmokeTest.php` +- Restore run detail post-execution proof/results rendering: `Spec335RestoreRunDetailProductizationSmokeTest.php` +- Review Pack management PDF / Stored Report output state: `Spec379ManagementReportPdfSmokeTest.php` + +Browser tests asserted no JavaScript errors and no unexpected console logs on the tested surfaces. + +## 11. Regression Proof + +Completed: + +- PostgreSQL schema/type/index/model/query proof passed. +- PostgreSQL rollback/down-up preservation proof passed. +- Existing `BackupItem` unit regression suite passed. +- Focused browser proof passed across evidence, operations, provider, backup, restore, review pack, stored report, and management PDF surfaces. +- No UI/runtime surface files were edited. +- No authorization, global search, provider registration, destructive/high-impact action, or asset behavior was changed. + +## 12. Staging Validation + +Staging/Dokploy validation is not accessible from this agent session. + +Required staging release checks before production: + +- Run the migration on staging PostgreSQL. +- Confirm app boot. +- Run the focused PostgreSQL Spec405 test or equivalent staging validation. +- Smoke the representative payload-backed surfaces. +- Confirm no secrets or raw provider credential payloads appear in logs/screenshots. + +Because staging is unavailable here, final readiness remains `PASS WITH CONDITIONS`. + +## 13. Remaining Findings + +- No P0 findings. +- No unresolved `json` column classification. +- No in-scope schema or application regression currently known after fixing rollback-compatible query handling and grouped table conversion. +- Remaining P1 condition: staging/Dokploy validation unavailable in this session. +- Browser proof passed locally. + +## 14. Deferred Items + +- Staging/Dokploy validation and migration runtime observation. +- Optional future online migration strategy if a production-sized table proves too large for direct `ALTER COLUMN`. +- Full browser/runtime audit remains out of scope. +- Governance artifact lifecycle and retention remain separate follow-up scope. + +## 15. Validation Commands + +Passed: + +```bash +git diff --check +cd apps/platform && ./vendor/bin/sail php vendor/bin/pest -c phpunit.pgsql.xml tests/Feature/Database/JsonbDataLayerHardeningTest.php +cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Unit/BackupItemTest.php +cd apps/platform && ./vendor/bin/sail pint app/Models/BackupItem.php database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php tests/Feature/Database/JsonbDataLayerHardeningTest.php +cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Browser/Spec403EvidenceCurrentnessRuntimeClosureSmokeTest.php tests/Browser/Spec394ProviderFreshnessPermissionSmokeTest.php tests/Browser/Spec371BackupSetProductizationSmokeTest.php tests/Browser/Spec335RestoreRunDetailProductizationSmokeTest.php tests/Browser/Spec379ManagementReportPdfSmokeTest.php +``` + +## 16. Product Surface, Filament, Deployment, and Recommended Next Step Close-Out + +- **Application implementation status**: implemented locally with focused PostgreSQL rollback/down-up, unit, formatter, diff, and browser proof passing; staging validation still conditions the gate. +- **Livewire v4 compliance**: Livewire 4.1.4 confirmed by Laravel Boost; no Livewire code changed. +- **Provider registration location**: Laravel 12 panel providers remain in `apps/platform/bootstrap/providers.php`; no provider registration changed. +- **Global search posture**: unchanged. No globally searchable resource was added or modified. +- **Destructive/high-impact actions**: none added or changed; no confirmation/authorization behavior changed. +- **Asset strategy**: no frontend assets added, no `FilamentAsset` registration, no new `filament:assets` deploy requirement beyond the existing deployment baseline. +- **Product Surface Impact**: no runtime UI surface changed. +- **UI Surface Impact**: none; focused browser proof is regression-only. +- **No-legacy posture**: canonical data-layer conversion; no compatibility shim or dual-write path introduced. +- **Page archetype / surface budgets / Technical Annex / deep-link demotion / canonical status vocabulary**: N/A for changed code; existing surfaces remain unchanged. +- **Product Surface exceptions**: none. +- **Focused browser proof**: passed across evidence/currentness, operations, provider readiness, backup, restore, review pack, stored report, and management PDF surfaces. +- **Human Product Sanity**: visible complexity unchanged; no raw payload exposure added; current/released/failed/partial semantics unchanged. +- **Implementation-report fields**: Livewire v4, provider registration, global search, destructive/high-impact actions, asset strategy, deployment impact, tests/browser result, and visible complexity are recorded here. +- **Deployment impact**: database migration only. No env vars, queues, scheduler, storage, routes, provider scopes, panels, assets, or worker changes. Staging validation is required before production promotion. +- **No completed-spec rewrite assertion**: Specs 400-404 were read-only context; their files were not rewritten. +- **Recommended next step**: run staging PostgreSQL migration validation before production promotion. diff --git a/specs/405-json-to-jsonb-data-layer-hardening/plan.md b/specs/405-json-to-jsonb-data-layer-hardening/plan.md new file mode 100644 index 00000000..002998a1 --- /dev/null +++ b/specs/405-json-to-jsonb-data-layer-hardening/plan.md @@ -0,0 +1,401 @@ +# Implementation Plan: Spec 405 - JSON-to-JSONB Data-layer Hardening + +**Branch**: `405-json-to-jsonb-data-layer-hardening` | **Date**: 2026-06-23 | **Spec**: `specs/405-json-to-jsonb-data-layer-hardening/spec.md` +**Input**: Feature specification from `specs/405-json-to-jsonb-data-layer-hardening/spec.md` + +## Summary + +Prepare and later implement a database-hardening slice that inventories every live PostgreSQL `json` and `jsonb` column, classifies all `json` columns, converts only appropriate queryable trust-layer payload columns to `jsonb`, adds only query-backed indexes, proves semantic payload preservation, and records local plus staging-like validation. The implementation must not add product behavior, UI surfaces, broad abstractions, normalized replacement tables, or speculative indexes. + +## Technical Context + +**Language/Version**: PHP 8.4.15, Laravel 12.52.0, Filament 5.2.1, Livewire 4.1.4. +**Primary Dependencies**: Laravel migrations/schema builder, PostgreSQL, Pest 4, Filament/Livewire test helpers where focused browser or rendered-surface proof is needed. +**Storage**: PostgreSQL via Sail/Dokploy; target change is existing column type conversion from `json` to `jsonb` where classified `CONVERT`. +**Testing**: Pest 4, PostgreSQL lane, targeted feature tests, focused browser smoke. +**Validation Lanes**: pgsql, confidence/feature, focused browser, optional profiling/explain for new indexes. +**Target Platform**: TenantPilot Laravel monolith under `apps/platform`; Spec Kit artifacts under `specs/405-json-to-jsonb-data-layer-hardening/`. +**Project Type**: Laravel web application plus Spec Kit workflow. +**Performance Goals**: Avoid speculative indexes; each new index must have an existing query path and bounded write-overhead rationale. +**Constraints**: No new product concepts, UI surfaces, Graph calls, authorization model, lifecycle behavior, normalized replacement tables, provider semantics, broad abstractions, or completed-spec rewrites. +**Scale/Scope**: All live PostgreSQL `json` and `jsonb` columns; conversion limited to selected existing columns with proof. + +## Repository Truth And Initial Signals + +Laravel Boost confirmed PostgreSQL and current packages: + +```text +PHP 8.4.15 +Laravel 12.52.0 +Filament 5.2.1 +Livewire 4.1.4 +Pest 4.3.1 +PostgreSQL +``` + +Laravel 12 migrations support `$table->jsonb('column')`; implementation should use Laravel migrations where possible and raw PostgreSQL `ALTER TABLE ... ALTER COLUMN ... TYPE jsonb USING ...::jsonb` where column type alteration requires explicit SQL. + +Live PostgreSQL schema inspection on 2026-06-23 found active `json` columns requiring classification in: + +```text +alert_deliveries.payload +alert_rules.tenant_allowlist +audit_logs.metadata +backup_items.assignments +backup_items.metadata +backup_items.payload +backup_schedules.days_of_week +backup_schedules.policy_types +backup_sets.metadata +managed_environment_onboarding_sessions.state +managed_environment_permissions.details +managed_environments.metadata +managed_environments.rbac_canary_results +managed_environments.rbac_last_warnings +policies.metadata +policy_versions.assignments +policy_versions.metadata +policy_versions.scope_tags +policy_versions.secret_fingerprints +policy_versions.snapshot +restore_runs.group_mapping +restore_runs.metadata +restore_runs.preview +restore_runs.requested_items +restore_runs.results +tenant_settings.value +workspace_settings.value +``` + +Newer trust-layer paths already use `jsonb`, including `operation_runs.summary_counts`, `operation_runs.failure_summary`, `operation_runs.context`, baseline `scope_jsonb`/`summary_jsonb`/`meta_jsonb`, evidence summaries, findings evidence, review pack summaries/options, stored report payloads, provider connection metadata/scopes, and review publication resolution payloads. + +## Technical Approach + +1. Record branch, HEAD, dirty state, and `git diff --check`. +2. Query PostgreSQL `information_schema.columns` for all `json` and `jsonb` columns and collect row counts, null counts, defaults, indexes, and constraints. +3. Map each column to model casts, factories/fixtures, query usages, Filament/rendered usage, tests, and sensitive-data boundaries. +4. Classify each column as `CONVERT`, `KEEP_JSON`, `ALREADY_JSONB`, `DEPRECATED`, or `DECISION_REQUIRED`. +5. Write one or more focused migrations converting only `CONVERT` columns. +6. Add only query-backed JSONB indexes with explicit proof and rollback/drop strategy. +7. Update casts or query code only if required by tests after conversion. +8. Add PostgreSQL/feature tests proving type conversion, semantic preservation, model read/write behavior, scope/authorization boundaries, and representative domain regressions. +9. Run focused browser proof over existing payload-backed surfaces. +10. Produce `implementation-report.md` with the inventory matrix, validation results, remaining findings, staging status, and final gate result. + +## Likely Affected Repository Surfaces + +Preparation identifies likely inspection and implementation surfaces; implementation must verify exact paths before editing. + +```text +apps/platform/database/migrations/ +apps/platform/app/Models/ +apps/platform/database/factories/ +apps/platform/tests/ +apps/platform/app/Support/ +apps/platform/app/Services/ +apps/platform/app/Jobs/ +apps/platform/app/Filament/ +apps/platform/resources/views/ +specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md +``` + +Likely model/cast inspection targets include: + +```text +App\Models\AlertDelivery +App\Models\AlertRule +App\Models\AuditLog +App\Models\BackupItem +App\Models\BackupSchedule +App\Models\BackupSet +App\Models\ManagedEnvironment +App\Models\ManagedEnvironmentOnboardingSession +App\Models\ManagedEnvironmentPermission +App\Models\Policy +App\Models\PolicyVersion +App\Models\RestoreRun +App\Models\TenantSetting +App\Models\WorkspaceSetting +``` + +## UI / Surface Guardrail Plan + +- **Guardrail scope**: backend data-layer hardening with focused rendered regression proof. +- **Affected routes/pages/actions/states/navigation/panel/provider surfaces**: none changed by implementation unless the spec is updated first. +- **No-impact class**: backend-only schema/storage conversion. +- **Native vs custom classification summary**: N/A. +- **Shared-family relevance**: evidence/report viewers, provider readiness, OperationRun proof, backup/restore proof, and audit metadata are regression consumers only. +- **State layers in scope**: persistence and existing rendered state regression only. +- **Audience modes in scope**: operator/MSP and customer/read-only only for regression proof; no disclosure behavior changes. +- **Decision/diagnostic/raw hierarchy plan**: unchanged; raw payloads remain technical/audit detail. +- **Raw/support gating plan**: unchanged. +- **One-primary-action / duplicate-truth control**: unchanged. +- **Handling modes by drift class or surface**: report-only unless conversion causes a confirmed regression, then minimal in-scope fix. +- **Repository-signal treatment**: review-mandatory for changed database/runtime paths. +- **Special surface test profiles**: backend storage conversion plus focused browser proof. +- **Required tests or manual smoke**: PostgreSQL migration tests, feature regressions, focused browser smoke. +- **Exception path and spread control**: any required UI edit is out of scope and must stop implementation for spec/plan update. +- **Active feature PR close-out entry**: Guardrail / database hardening. +- **UI/Productization coverage decision**: No UI surface impact. +- **Coverage artifacts to update**: none unless implementation unexpectedly changes rendered UI; then stop and update spec/plan first. +- **No-impact rationale**: storage type conversion and proof only. +- **Navigation / Filament provider-panel handling**: no panel/provider changes. +- **Screenshot or page-report need**: focused browser proof may produce screenshots/logs as evidence; no page report required without UI changes. + +## Product Surface Contract Plan + +- **Product Surface Contract reference**: `docs/product/standards/product-surface-contract.md` as regression lens only. +- **No-legacy posture**: canonical replacement; no compatibility shims or dual-write paths. +- **Page archetype and surface budget plan**: N/A for changed code; browser proof names existing inspected archetypes. +- **Technical Annex and deep-link demotion plan**: unchanged; conversion must not expose raw payloads, internal IDs, OperationRun links, or evidence deep links by default. +- **Canonical status vocabulary plan**: unchanged. +- **Product Surface exceptions**: none. +- **Browser verification plan**: focused existing-surface regression proof required. +- **Human Product Sanity plan**: final implementation report confirms unchanged trust semantics. +- **Visible complexity outcome target**: neutral. +- **Implementation report target**: `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`. + +## Filament / Livewire / Deployment Posture + +- **Livewire v4 compliance**: Livewire 4.1.4 confirmed; no Livewire code change planned. +- **Panel provider registration location**: Laravel 12 panel providers remain in `apps/platform/bootstrap/providers.php`; no panel change. +- **Global search posture**: no resource global search posture changed. If implementation touches a resource unexpectedly, verify View/Edit/global search safety or stop for spec update. +- **Destructive/high-impact action posture**: no actions added or changed. Existing destructive/high-impact action proof from Specs 401-404 must not regress. +- **Asset strategy**: no assets, no `FilamentAsset` registration, no new `filament:assets` requirement beyond existing deployment baseline. +- **Testing plan**: database/migration tests, feature/domain regressions, focused browser proof for existing payload-backed surfaces. +- **Deployment impact**: migrations and staging/Dokploy validation. No env vars, queues, scheduler, storage, assets, routes, provider scopes, or panel providers planned. + +## Shared Pattern & System Fit + +- **Cross-cutting feature marker**: yes at data storage level. +- **Systems touched**: database schema, existing casts, existing query paths, tests, implementation report. +- **Shared abstractions reused**: Laravel migrations, existing models/casts, existing scoped query paths, existing tests/browser fixtures. +- **New abstraction introduced? why?**: none planned. +- **Why the existing abstraction was sufficient or insufficient**: existing models and services already own payload meaning; storage conversion does not require a new runtime layer. +- **Bounded deviation / spread control**: any new helper must be justified as a narrow test/support helper, not a runtime framework. + +## OperationRun UX Impact + +- **Touches OperationRun start/completion/link UX?**: no. +- **Central contract reused**: N/A. +- **Delegated UX behaviors**: N/A. +- **Surface-owned behavior kept local**: N/A. +- **Queued DB-notification policy**: N/A. +- **Terminal notification path**: N/A. +- **Exception path**: none. + +## Provider Boundary & Portability Fit + +- **Shared provider/platform boundary touched?**: storage inspection only. +- **Provider-owned seams**: provider raw/permission/readiness payload keys remain unchanged. +- **Platform-core seams**: storage type, workspace/managed-environment scope, audit/report/evidence ownership, migration safety. +- **Neutral platform terms / contracts preserved**: workspace, managed environment, provider, connection, operation, evidence, report, backup, restore. +- **Retained provider-specific semantics and why**: existing Microsoft/Intune payload keys remain provider-owned payload content. +- **Bounded extraction or follow-up path**: none for this spec. + +## Domain / Model Implications + +- No new model, table, persisted artifact, enum, status family, route, action, provider type, or source of truth is introduced. +- Existing model casts may remain unchanged because Laravel treats `json` and `jsonb` similarly at the application layer; implementation must update casts only if tests prove current behavior breaks. +- Existing raw payload keys and product meaning remain unchanged. +- Columns with unclear ownership or product semantics must be classified `DECISION_REQUIRED`, not converted by assumption. + +## Data / Migration Implications + +Migration pattern for direct conversions: + +```sql +ALTER TABLE table_name +ALTER COLUMN column_name TYPE jsonb +USING column_name::jsonb; +``` + +Rollback pattern where feasible: + +```sql +ALTER TABLE table_name +ALTER COLUMN column_name TYPE json +USING column_name::json; +``` + +Implementation must preserve: + +- nullable state +- defaults +- constraints +- existing indexes unless intentionally replaced +- row counts +- application read/write behavior + +Implementation must document rollback limitations: + +- `jsonb` normalizes key order +- duplicate JSON object keys may be normalized +- semantic content must remain preserved, but textual representation may differ + +## Index Strategy + +Allowed only with existing query proof: + +- GIN index for existing containment/query usage. +- Expression index for existing frequently queried JSONB key. +- Partial index for existing filtered JSONB key access. + +Rejected: + +- index every converted column +- index because `jsonb` supports it +- index for future lifecycle/export/dashboard guesses +- index without write-overhead note + +Each index requires a row in the implementation report: + +```text +Index | Table/Column | Query Path | Reason | Expected Benefit | Write Overhead Risk | Proof +``` + +## RBAC / Security / Audit Implications + +- No RBAC behavior changes. +- Any changed query involving payload keys must retain existing workspace and managed-environment scoping. +- Non-member and wrong-scope access must remain deny-as-not-found where applicable. +- Customer-safe report/review/evidence paths must not expose raw payloads because of conversion or debug output. +- Reports, logs, screenshots, and test fixtures must not include secrets, tokens, raw credential payloads, sensitive raw provider payloads, or customer-sensitive raw payloads. +- Audit metadata conversion must preserve actor/context fields. + +## OperationRun / Evidence / Result Truth Implications + +The plan distinguishes: + +- **Execution truth**: existing `OperationRun` status/outcome/summary/context; live columns are already `jsonb`, but regression proof still checks rendering. +- **Artifact truth**: `ReviewPack`, `StoredReport`, evidence snapshots/items, backup sets/items. +- **Backup/snapshot truth**: policy versions, backup payloads, baseline/evidence payloads. +- **Recovery/evidence truth**: restore previews/results, evidence currentness, report receipts. +- **Operator next action**: unchanged existing UI states and actions. + +## Test Strategy + +Required test groups: + +1. PostgreSQL schema/type tests for every converted column. +2. Migration semantic preservation tests for representative non-sensitive payloads. +3. Model cast/read-write tests where converted columns are written by Eloquent models. +4. Query-path tests for any changed JSON key query and any new index. +5. Evidence/currentness regression tests for converted evidence/report/review payloads. +6. OperationRun/audit regression tests where summary/context/audit metadata is in scope. +7. Provider readiness/freshness/permission regression tests where provider/environment payloads are converted. +8. Backup/restore payload regression tests for backup items/sets/schedules and restore preview/results. +9. Review/report receipt regression tests for review pack/stored report/customer output. +10. Authorization/scope tests for changed payload queries. +11. Focused browser smoke for representative existing payload-backed pages. + +Preferred validation commands: + +```bash +cd apps/platform && ./vendor/bin/sail php vendor/bin/pest -c phpunit.pgsql.xml --filter=Spec405 +cd apps/platform && ./vendor/bin/sail artisan test --filter=Evidence +cd apps/platform && ./vendor/bin/sail artisan test --filter=OperationRun +cd apps/platform && ./vendor/bin/sail artisan test --filter=Provider +cd apps/platform && ./vendor/bin/sail artisan test --filter=Backup +cd apps/platform && ./vendor/bin/sail artisan test --filter=Restore +cd apps/platform && ./vendor/bin/sail artisan test --filter=ReviewPack +cd apps/platform && ./vendor/bin/sail artisan test --filter=StoredReport +``` + +Use narrower commands where implementation creates focused Spec 405 tests. + +## Rollout And Deployment Considerations + +- Local: run through Laravel Sail against PostgreSQL. +- Staging: validate migration execution, app boot, queue/browser relevant proof, representative pages, and rollback/forward notes where safe. +- Production: do not claim production readiness unless staging-like validation passes or the final report explicitly records `PASS WITH CONDITIONS`. +- Migrations: assess table size/row count and lock risk before direct conversion. +- Env vars: none expected. +- Queues/scheduler/storage/assets: none expected, but affected flows may rely on existing workers/storage and must not regress. +- Dokploy: database migration and app boot validation required before production promotion. + +## Risk Controls + +- Stop before migration if inventory is incomplete. +- Stop before conversion if a high-risk column has unclear product ownership. +- Use semantic JSON comparisons, not raw string comparisons. +- Classify large-table conversion risk before direct `ALTER COLUMN`. +- Do not add speculative indexes. +- Do not print sensitive payload samples. +- Do not rewrite completed specs. +- If a conversion causes UI/rendered regression, fix the data-layer cause if bounded; otherwise stop for spec update. + +## Constitution Check + +- Inventory-first: PASS; inventory and source-of-truth classification precede conversion. +- Read/write separation: PASS; no Graph/write product behavior is added. +- Graph contract path: PASS; no Graph calls are added. +- Deterministic capabilities: N/A; no capability resolver changes. +- RBAC-UX: PASS; authorization must not change and changed payload queries require scope tests. +- Workspace isolation: PASS; scoped query proof required. +- Tenant/managed-environment isolation: PASS; managed-environment scope remains enforced. +- Run observability: N/A; no new OperationRun. +- OperationRun start UX: N/A. +- Ops-UX lifecycle/summary counts: no changes to lifecycle; current `operation_runs` JSONB posture is inspected. +- Data minimization: PASS; sensitive payloads are not dumped. +- Test governance: PASS; PostgreSQL and focused browser proof are explicit. +- Proportionality: PASS; storage conversion only, no new product truth. +- No premature abstraction: PASS; no new framework. +- Persisted truth: PASS; no new persisted entity/table. +- Behavioral state: PASS; no new state. +- UI semantics: PASS; no UI semantics added. +- Shared pattern first: PASS; existing models/services/tests are reused. +- Provider boundary: PASS; provider payload keys unchanged. +- V1 explicitness / few layers: PASS. +- Spec discipline / bloat check: PASS; one coherent data-layer hardening package. +- Product Surface Contract Gate: PASS as no rendered surface change plus focused regression proof. + +## Test Governance Check + +- **Test purpose / classification by changed surface**: PostgreSQL migration/type proof, feature/domain regression, focused browser smoke. +- **Affected validation lanes**: pgsql, confidence, browser. +- **Why this lane mix is the narrowest sufficient proof**: storage type is PostgreSQL-specific and payload-backed pages require rendered regression proof, but no full browser audit is needed. +- **Narrowest proving command(s)**: focused Spec 405 pgsql/feature/browser tests once created. +- **Fixture / helper / factory / seed / context cost risks**: minimal explicit fixtures only; no shared default broadening. +- **Expensive defaults or shared helper growth introduced?**: no planned. +- **Heavy-family additions, promotions, or visibility changes**: focused browser proof only. +- **Surface-class relief / special coverage rule**: no UI code changes; browser is regression proof. +- **Closing validation and reviewer handoff**: verify inventory coverage, migration proof, no sensitive payload output, no speculative indexes, and final gate result. +- **Budget / baseline / trend follow-up**: record if pgsql/browser runtime materially increases. +- **Review-stop questions**: Is every `json` column classified? Is every conversion justified? Does every new index have existing query proof? Did any UI or product behavior change? Was staging-like validation completed or properly conditioned? +- **Escalation path**: document-in-feature for bounded findings; follow-up-spec for unsafe conversion, online migration need, or unresolved product/schema decision. +- **Active feature PR close-out entry**: Guardrail / Database Hardening. +- **Why no dedicated follow-up spec is needed**: the slice is bounded unless implementation finds large-table online migration or unresolved product/schema ownership. + +## Implementation Phases + +### Phase 1 - Inventory and Classification + +Build schema inventory from PostgreSQL, migrations, models, casts, factories, query paths, and tests. Produce the draft inventory matrix before any migration. + +### Phase 2 - Conversion and Index Design + +Write focused migration(s) only for `CONVERT` columns. Add no index unless tied to an existing query path. Keep rollback explicit. + +### Phase 3 - Regression and Preservation Proof + +Add PostgreSQL and feature tests proving column type, semantic preservation, read/write behavior, scope/authorization, and representative domain behavior. + +### Phase 4 - Browser, Staging, and Report + +Run focused browser proof, staging-like validation where available, final dirty-state checks, and complete the implementation report with gate result. + +## Complexity Tracking + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|--------------------------------------| +| None planned | N/A | N/A | + +## Proportionality Review + +- **Current operator problem**: trust-layer payload storage is inconsistent and can undermine future evidence/report/lifecycle work. +- **Existing structure is insufficient because**: older `json` payload columns conflict with current PostgreSQL/jsonb posture and query/index expectations. +- **Narrowest correct implementation**: classify all columns, convert only selected existing columns, add only query-backed indexes, and prove behavior. +- **Ownership cost created**: migration review, PostgreSQL tests, focused browser proof, and implementation report. +- **Alternative intentionally rejected**: blanket conversion plus blanket indexes. +- **Release truth**: current-release data-layer readiness. diff --git a/specs/405-json-to-jsonb-data-layer-hardening/spec.md b/specs/405-json-to-jsonb-data-layer-hardening/spec.md new file mode 100644 index 00000000..ab0d5d83 --- /dev/null +++ b/specs/405-json-to-jsonb-data-layer-hardening/spec.md @@ -0,0 +1,441 @@ +# Feature Specification: Spec 405 - JSON-to-JSONB Data-layer Hardening + +**Feature Branch**: `405-json-to-jsonb-data-layer-hardening` +**Created**: 2026-06-23 +**Status**: Draft / Ready for implementation preparation review +**Type**: Database hardening / migration proof / data integrity spec +**Runtime posture**: Data-layer conversion and proof only. No new product behavior, no new product concepts, and no new rendered UI surfaces. +**Input**: User-provided "Spec 405 - JSON-to-JSONB Data-layer Hardening" draft from `/Users/ahmeddarrazi/.codex/attachments/b7628b1e-f536-40b3-8364-0f6f476c59ac/pasted-text.txt`, current repo truth, Spec 400 product-contract audit context, Specs 401-404 implementation/proof context, roadmap/spec-candidate queue, and live PostgreSQL schema inspection through Laravel Boost. + +## Candidate Selection Context + +- **Selected candidate**: JSON-to-JSONB Data-layer Hardening. +- **Source location**: Direct user-provided Spec 405 draft, promoted from the Spec 400 P1 data-layer concern that important structured payload columns still use PostgreSQL `json` even though the project standard prefers `jsonb` for queryable policy, evidence, backup, restore, report, provider, and audit payloads. +- **Why selected**: `docs/product/spec-candidates.md` currently reports no safe automatic next-best-prep target, but the operator supplied a concrete manual candidate. The live PostgreSQL schema confirms active `json` columns remain in trust-layer tables such as `policy_versions`, `backup_items`, `restore_runs`, `audit_logs`, `alert_deliveries`, `managed_environment_permissions`, settings tables, and provider/environment metadata paths, while newer foundations already use `jsonb`. +- **Roadmap relationship**: Supports current governance and architecture hardening by reducing data-layer ambiguity before governance artifact lifecycle/retention, report-output expansion, or broader runtime/browser audit work proceeds. +- **Close alternatives deferred**: + - `governance-artifact-lifecycle-retention-runtime`: broader P2 lifecycle semantics; must not be hidden inside a storage-type conversion. + - `provider-readiness-onboarding-productization`: optional UX/productization work; unrelated to payload storage type proof. + - `cross-domain-indicator-runtime-follow-through`: semantic runtime adoption; not a database hardening gate. + - `manual-system-panel-browser-fixture-or-audit-procedure`: validation procedure work; not a payload storage hardening slice. + - Reopening Specs 400-404 or Specs 378-404: forbidden. They are read-only context and proof lineage only. +- **Completed-spec guardrail result**: + - No `specs/405-json-to-jsonb-data-layer-hardening/` package existed before this preparation. + - A local and remote branch named `405-dach-trust-datenschutz-security-website-surface` existed before preparation, but no TenantPilot `specs/405-*` package existed. The operator explicitly supplied Spec ID 405, so this package uses the requested number and records the branch-prefix collision as preparation context. + - Specs 400, 401, 402, 403, and 404 contain audit, implementation-report, validation, browser, or completed-task signals and are read-only historical context. Their close-out notes, validation results, completed task markers, screenshots, smoke history, and implementation reports must not be rewritten, normalized, unchecked, or removed. +- **Smallest viable implementation slice**: Inventory every PostgreSQL `json` and `jsonb` column, classify every `json` column, convert only appropriate queryable trust-layer `json` columns to `jsonb`, add only query-backed indexes, prove semantic payload preservation and existing behavior, run PostgreSQL migration/rollback validation, run focused regression tests and representative browser proof for payload-backed surfaces, and record a final JSON/JSONB inventory matrix plus gate result. +- **Feature description for Spec Kit**: Prove and harden TenantPilot structured payload storage by inventorying all PostgreSQL `json` and `jsonb` columns, converting only appropriate queryable trust-layer `json` columns to `jsonb`, preserving payload semantics and scope boundaries, and documenting local plus staging-like validation without adding new product behavior or UI surfaces. + +## Spec Candidate Check *(mandatory - SPEC-GATE-001)* + +- **Problem**: TenantPilot stores critical structured proof and operational metadata in a mix of PostgreSQL `json` and `jsonb`; important older payload columns are still `json` even though the project standard prefers `jsonb` for queryable snapshots, backup/restore payloads, evidence/report metadata, and audit context. +- **Today's failure**: Future governance, evidence, report, backup, restore, and lifecycle work can build on inconsistent storage assumptions, miss query/index opportunities, or let agents guess which payloads are queryable/trust-layer data versus low-risk configuration values. +- **User-visible improvement**: Operators and reviewers get stronger confidence that evidence, backup, restore, report, provider, audit, and OperationRun-adjacent payloads remain semantically preserved while becoming query-ready where the product already depends on structured payload proof. +- **Smallest enterprise-capable version**: A schema inventory, explicit classification matrix, targeted `json` to `jsonb` migrations with rollback notes, query-backed indexes only, semantic preservation tests, PostgreSQL lane validation, focused browser proof for representative payload-backed surfaces, and a spec-local implementation report. +- **Explicit non-goals**: No new evidence semantics, provider state model, backup/restore feature, governance lifecycle/retention/export/delete/hold behavior, report template, UI surface, navigation, authorization model, broad normalization, event-sourcing redesign, or completed-spec rewrite. +- **Permanent complexity imported**: One spec package, one future implementation report/inventory matrix, targeted migrations, focused tests, and possibly justified indexes. No new persisted entity/table, enum/status family, runtime abstraction, product vocabulary, UI framework, or provider framework is approved. +- **Why now**: Spec 400 identified the data-layer gap as a P1 condition, Specs 401-404 closed adjacent action, authorization, evidence/currentness, and management-report runtime proof gates, and the live schema still shows older `json` payload columns in trust-layer areas. +- **Why not local**: Converting one column locally would not prove all payload columns were classified, excluded columns were intentional, indexes were query-backed, and report/evidence/backup/restore/provider/audit behavior remained unchanged. +- **Approval class**: Core Enterprise. +- **Red flags triggered**: Foundation/hardening language and many tables. Defense: the slice is bounded to existing storage types, existing payload semantics, and proof; it forbids new models, product concepts, broad frameworks, speculative indexes, and UI changes. +- **Score**: Nutzen: 2 | Dringlichkeit: 2 | Scope: 2 | Komplexitaet: 1 | Produktnaehe: 1 | Wiederverwendung: 2 | **Gesamt: 10/12** +- **Decision**: approve as a bounded data-layer hardening and proof package. + +## Problem Statement + +TenantPilot depends on structured payloads for policy snapshots, backup items, restore previews/results, audit metadata, provider permission/readiness details, evidence summaries, report receipts, review packs, settings, alerts, and OperationRun-adjacent proof. Some of these payloads remain PostgreSQL `json`, while newer code already uses `jsonb` for similar trust-layer data. + +This spec answers: + +```text +Can TenantPilot safely convert appropriate important json payload columns to jsonb without data loss, product behavior changes, authorization drift, evidence/currentness drift, report/restore/backup regressions, or speculative indexing? +``` + +A conversion is successful only when every `json` column was intentionally classified, selected conversions preserve semantic payload content, excluded columns have explicit reasons, migrations and rollback behavior are documented, PostgreSQL validation passes, and existing trust-layer behavior remains unchanged. + +## Product / Business Value + +- Reduces future data-layer ambiguity around queryable governance, evidence, backup, restore, provider, report, and audit payloads. +- Makes later lifecycle/retention/export/reporting work smaller and safer by establishing clear payload storage posture first. +- Prevents false confidence from application casts alone; storage type, query paths, indexes, tests, and staging-like validation must agree. +- Keeps trust-layer payload semantics stable while hardening the substrate. + +## Primary Users / Operators + +- Product owner and release reviewer deciding whether the platform is ready for lifecycle/retention and customer-output expansion. +- Engineering reviewer validating migration safety, rollback posture, and test coverage. +- MSP/operator relying on backup, restore, evidence, provider, report, and audit proof. +- Support/platform operator diagnosing payload-backed behavior without raw sensitive payload leakage. + +## Spec Scope Fields *(mandatory)* + +- **Scope**: database schema inventory, classification, migrations, indexes, tests, validation, focused browser proof, and final data-layer implementation report. +- **Primary Routes / Surfaces**: No route or UI surface is added or changed. Focused browser proof later inspects representative existing payload-backed surfaces only, such as Evidence Overview, Monitoring -> Operations, Provider readiness/detail, Backup/Restore proof, Review Pack or Stored Report output. +- **Data Ownership**: + - Workspace-owned payloads remain workspace scoped. + - Managed-environment-owned payloads remain workspace plus managed-environment scoped. + - Customer-safe report/review payloads remain bound to existing `ReviewPack`, `StoredReport`, `EnvironmentReview`, and evidence ownership rules. + - No new persisted entity/table/artifact is introduced. +- **RBAC**: + - No authorization behavior changes are allowed. + - Existing policies, gates, customer-output gates, deny-as-not-found behavior, and global search posture must continue to pass. + - Any payload query changed during implementation must preserve workspace and managed-environment scoping. + +For canonical-view or mixed-scope surfaces: + +- **Default filter behavior when environment-context is active**: unchanged. Payload storage conversion must not alter route-owned workspace/environment context or explicit page filters. +- **Explicit entitlement checks preventing cross-tenant leakage**: any changed JSON key query must retain existing scoped query predicates before or with the payload predicate; tests must cover changed high-risk query paths. + +## No Legacy / No Backward Compatibility Constraint *(mandatory)* + +TenantPilot is pre-production for this data-layer hardening. + +- **Compatibility posture**: canonical storage hardening with current payload semantics preserved. +- **Legacy aliases, fallback readers, hidden routes, duplicate UI, old labels, or historical fixtures kept?**: no new compatibility aliases, dual-write paths, fallback readers, hidden routes, duplicate UI, or legacy fixtures are allowed unless this spec is updated with an explicit exception. +- **Why clean replacement is safe now**: PostgreSQL `json` columns already contain valid JSON. Conversion to `jsonb` is a storage hardening change, and semantic behavior must be proven through decoded JSON equality/key checks rather than raw string order. + +## UI Surface Impact *(mandatory - UI-COV-001)* + +Does this spec add, remove, rename, or materially change any reachable UI surface? + +- [x] No UI surface impact +- [ ] Existing page changed +- [ ] New page/route added +- [ ] Navigation changed +- [ ] Filament panel/provider surface changed +- [ ] New modal/drawer/wizard/action added +- [ ] New table/form/state added +- [ ] Customer-facing surface changed +- [ ] Dangerous action changed +- [ ] Status/evidence/review presentation changed +- [ ] Workspace/environment context presentation changed + +## UI/Productization Coverage + +N/A - no reachable UI surface impact. + +- **No-impact rationale**: This spec changes only storage type, migrations, proof, and tests. It may require focused browser proof to show existing payload-backed pages still render correctly after conversion, but it must not edit Filament resources/pages/widgets, Livewire components, Blade views, CSS, JavaScript, navigation, routes, modals, actions, tables, forms, or rendered copy. + +## Product Surface Impact *(mandatory for UI-affecting specs; otherwise N/A)* + +Reference: `docs/product/standards/product-surface-contract.md`. + +- **Product Surface Contract applies?**: no rendered product surface is changed. The contract is used only as a regression lens because payload conversion supports evidence, provider, restore, backup, report, and OperationRun proof surfaces. +- **Page archetype**: N/A for implementation changes. Browser proof later names inspected existing archetypes. +- **Primary user question**: N/A for runtime UI; regression question is "Do existing payload-backed surfaces still render truthful state after conversion?" +- **Primary action**: N/A. +- **Surface budget result**: N/A. +- **Technical Annex / deep-link demotion**: unchanged. Raw payloads, IDs, OperationRun links, evidence deep links, source keys, and provider payloads must not become more visible. +- **Canonical status vocabulary**: unchanged. +- **Visible complexity impact**: neutral. +- **Product Surface exceptions**: none. + +## Browser Verification Plan *(mandatory)* + +- **Browser proof required?**: yes during implementation, as backend regression proof for existing payload-backed surfaces. +- **No-browser rationale**: N/A for implementation because conversion affects storage used by existing rendered trust surfaces, even though no UI file is changed. +- **Focused path when required**: + 1. Evidence overview or evidence anchor surface renders the expected current/stale/missing proof state. + 2. Monitoring -> Operations or OperationRun detail renders summary/context/failure proof correctly. + 3. Provider detail/readiness/freshness surface renders provider metadata/permission state correctly. + 4. Backup or restore proof surface renders preview/result/payload-backed state correctly. + 5. Review Pack, Stored Report, or report receipt surface renders evidence/report payload state correctly. +- **Primary interaction to execute**: read existing pages, inspect payload-backed status and links, verify no 500, Livewire, Filament, console, or network errors, and verify no raw/internal payload exposure. +- **Console, Livewire, Filament, network, and 500-error checks**: required for focused browser proof. +- **Full-suite failure triage**: unrelated browser failures may be documented only when the focused Spec 405 path is green and evidence supports the classification. + +## Human Product Sanity Check *(mandatory)* + +- **Required?**: yes for final implementation report, focused on unchanged trust semantics rather than new UI design. +- **No-human-sanity rationale**: no new rendered surface, but human sanity must confirm the change did not make existing payload-backed surfaces claim stronger proof, expose raw data, or confuse current/released/failed/partial states. +- **Reviewer questions**: Are payload semantics unchanged? Are proof/currentness states still truthful? Are technical details still demoted? Are authorization/customer-safe boundaries unchanged? Does the final report avoid overclaiming staging/production readiness? +- **Planned result location**: `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`. + +## Product Surface Merge Gate Checklist *(mandatory)* + +- [x] No-legacy posture or approved exception recorded. +- [x] Product Surface Impact is `N/A - no rendered product surface changed` with rationale. +- [x] Browser proof is planned as focused regression proof for existing payload-backed surfaces. +- [x] Human Product Sanity is planned for unchanged trust semantics. +- [x] Product Surface exceptions are `none`. +- [x] Implementation report will state tests/browser result, Livewire v4 compliance, provider registration location, global search posture, destructive/high-impact action posture, asset strategy, deployment impact, visible complexity outcome, and no completed-spec rewrite assertion. + +## Cross-Cutting / Shared Pattern Reuse + +- **Cross-cutting feature?**: yes at the data layer; no runtime interaction family is implemented. +- **Interaction class(es)**: evidence/report viewers, OperationRun proof, provider readiness, backup/restore proof, audit metadata, alert delivery payloads, and settings payloads are storage consumers only. +- **Systems touched**: database migrations, model casts only where required, query paths only where required, tests, and implementation report. +- **Existing pattern(s) to extend**: Laravel migrations, PostgreSQL information schema inspection, existing model casts, existing scoped query paths, existing Pest PostgreSQL lane, existing browser smoke patterns. +- **Shared contract / presenter / builder / renderer to reuse**: N/A - no UI/runtime interaction contract is added. +- **Why the existing shared path is sufficient or insufficient**: Existing payload consumers are sufficient; the spec hardens storage beneath them. Any missing query path or cast defect must be fixed locally, not by creating a new abstraction. +- **Allowed deviation and why**: none planned. +- **Consistency impact**: payload meaning, scope, authorization, report/evidence truth, and audit metadata must stay consistent before and after conversion. +- **Review focus**: no speculative indexes, no data loss, no raw payload leakage, no broad normalization, no customer-output semantics change, and no completed-spec rewrite. + +## OperationRun UX Impact + +- **Touches OperationRun start/completion/link UX?**: no UX change. Existing `operation_runs` columns are already `jsonb` in the live schema, but OperationRun-adjacent proof paths must be inspected and regression-tested. +- **Shared OperationRun UX contract/layer reused**: N/A. +- **Delegated start/completion UX behaviors**: N/A. +- **Local surface-owned behavior that remains**: N/A. +- **Queued DB-notification policy**: N/A. +- **Terminal notification path**: N/A. +- **Exception required?**: none. + +## Provider Boundary / Platform Core Check + +- **Shared provider/platform boundary touched?**: yes at storage inspection level only. +- **Boundary classification**: mixed storage consumers; provider-specific payload semantics remain provider-owned, while storage and scope rules are platform-core. +- **Seams affected**: provider connection metadata, managed-environment permission details, policy snapshots/versions, backup/restore payloads, report/evidence payloads, and audit metadata. +- **Neutral platform terms preserved or introduced**: payload, metadata, evidence, operation, workspace, managed environment, provider connection, report, backup, restore. +- **Provider-specific semantics retained and why**: Microsoft/Intune keys inside existing payloads remain unchanged because the spec does not rename or reinterpret payload keys. +- **Why this does not deepen provider coupling accidentally**: storage type conversion does not introduce provider-specific platform contracts, labels, or taxonomies. +- **Follow-up path**: broader provider readiness/productization remains a separate manual candidate if later evidence requires it. + +## UI / Surface Guardrail Impact + +N/A - no operator-facing surface change. Focused browser proof later verifies existing surfaces still render after backend conversion. + +## Decision-First Surface Role + +N/A - no surface changed. + +## Audience-Aware Disclosure + +N/A - no disclosure changed. Tests and browser proof must still confirm raw/internal payloads do not become default-visible or customer-visible because of conversion or query changes. + +## UI/UX Surface Classification + +N/A - no surface changed. + +## Operator Surface Contract + +N/A - no new or materially refactored operator-facing page. + +## Proportionality Review *(mandatory when structural complexity is introduced)* + +- **New source of truth?**: no. +- **New persisted entity/table/artifact?**: no new entity/table. Migrations may alter storage type of existing columns only. +- **New abstraction?**: no. +- **New enum/state/reason family?**: no. +- **New cross-domain UI framework/taxonomy?**: no. +- **Current operator problem**: trust-layer payload storage is inconsistent and makes later evidence/report/lifecycle work riskier. +- **Existing structure is insufficient because**: leaving older high-risk payloads as `json` conflicts with the repo's PostgreSQL/jsonb posture and makes query/index proof ambiguous. +- **Narrowest correct implementation**: classify all JSON columns, convert only selected existing columns, add only query-backed indexes, and prove semantic preservation. +- **Ownership cost**: migration review, rollback notes, PostgreSQL lane tests, focused regression/browser proof, and final inventory/report maintenance inside the feature. +- **Alternative intentionally rejected**: blanket conversion of every `json` column and blanket GIN indexes. Both would import unnecessary migration risk and write overhead without product proof. +- **Release truth**: current-release data-layer hardening required before broader lifecycle/reporting work. + +## Testing / Lane / Runtime Impact *(mandatory for runtime behavior changes)* + +- **Test purpose / classification**: Feature, PostgreSQL, and Browser regression. Unit tests only for pure helpers if implementation introduces a bounded helper for semantic comparison or schema inspection. +- **Validation lane(s)**: PostgreSQL lane for migrations/jsonb/indexes; confidence/feature lane for payload behavior; focused browser lane for rendered regression proof. +- **Why this classification and these lanes are sufficient**: SQLite cannot prove PostgreSQL `jsonb`, GIN/expression indexes, or type conversion. Feature tests prove Laravel casts and behavior; browser proof proves representative existing pages still render. +- **New or expanded test families**: one focused Spec 405 test set only; avoid broad browser audit and heavy governance expansion. +- **Fixture / helper cost impact**: minimal explicit fixtures for selected converted columns; no global seed/default context broadening. +- **Heavy-family visibility / justification**: browser proof is explicit because payload storage backs critical trust surfaces; it must stay focused and not become a full UI audit. +- **Special surface test profile**: backend data-layer hardening with existing rendered trust-surface smoke. +- **Standard-native relief or required special coverage**: no Filament component changes; browser proof is regression-only. +- **Reviewer handoff**: verify every converted column appears in the inventory matrix, every excluded `json` column has a reason, every new index has query proof, and no test or fixture leaks sensitive payloads. +- **Budget / baseline / trend impact**: possible PostgreSQL lane runtime increase; document if material. +- **Escalation needed**: document-in-feature unless a conversion is unsafe or requires product/schema decision, then follow-up-spec. +- **Active feature PR close-out entry**: Guardrail / database hardening. +- **Planned validation commands**: + - `git status --short --branch` + - `git diff --check` + - `cd apps/platform && ./vendor/bin/sail artisan migrate` + - `cd apps/platform && ./vendor/bin/sail artisan migrate:rollback --step=1` only in a disposable local/test database when safe + - `cd apps/platform && ./vendor/bin/sail php vendor/bin/pest -c phpunit.pgsql.xml --filter=Spec405` + - targeted feature tests for evidence, OperationRun, provider, backup, restore, review/report payload behavior + - focused browser smoke for representative payload-backed surfaces + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Inventory and classify structured payload columns (Priority: P1) + +As an engineering reviewer, I need a complete inventory and classification of all `json` and `jsonb` columns so I can verify that storage decisions are intentional rather than guessed from column names. + +**Why this priority**: No migration should be written until every `json` column has a reasoned decision. + +**Independent Test**: Run the information-schema query and compare it to the implementation report matrix; every live `json` or `jsonb` column appears once with action, reason, risk, model/cast, query usage, row count, null/default details, and validation requirement. + +**Acceptance Scenarios**: + +1. **Given** the live PostgreSQL schema contains `json` and `jsonb` columns, **When** the implementation report is reviewed, **Then** every column is present with one of `CONVERT`, `KEEP_JSON`, `ALREADY_JSONB`, `DEPRECATED`, or `DECISION_REQUIRED`. +2. **Given** a high-risk payload column remains `json`, **When** the report is reviewed, **Then** it has an explicit reason and risk classification rather than being omitted. + +### User Story 2 - Convert selected trust-layer payloads safely (Priority: P1) + +As an operator and release reviewer, I need selected important payload columns converted to `jsonb` without semantic data loss so evidence, backup, restore, provider, report, and audit behavior remains trustworthy. + +**Why this priority**: Conversion is the core hardening value and must be proven before follow-up lifecycle or reporting work depends on queryable payloads. + +**Independent Test**: Run the migration against PostgreSQL, compare semantic JSON content before/after for selected samples, verify column types, null/default preservation, and rollback notes. + +**Acceptance Scenarios**: + +1. **Given** selected columns contain representative payloads, **When** the migration runs, **Then** decoded JSON content and required keys are preserved after conversion to `jsonb`. +2. **Given** a column has nulls or a default, **When** the migration runs, **Then** nullable/default behavior remains unchanged unless the implementation report records a spec-approved reason. +3. **Given** a large or unclear table cannot be safely converted directly, **When** classification is completed, **Then** the column is marked `DECISION_REQUIRED`; the final gate may be `PASS WITH CONDITIONS`, but the column is not forced through a speculative migration. + +### User Story 3 - Prove existing behavior and report readiness honestly (Priority: P2) + +As a product owner, I need a final implementation report that proves application behavior did not regress and clearly states whether data-layer readiness is `PASS`, `PASS WITH CONDITIONS`, or `FAIL`. + +**Why this priority**: Storage conversion is not complete unless runtime behavior, authorization/scope boundaries, and representative rendered surfaces still work. + +**Independent Test**: Run targeted feature/PostgreSQL tests, focused browser proof, and final dirty-state checks; verify the implementation report records commands, results, remaining findings, and staging-like validation status. + +**Acceptance Scenarios**: + +1. **Given** converted payloads support evidence/provider/backup/restore/report/audit flows, **When** targeted tests and browser proof run, **Then** existing behavior remains unchanged and no raw/internal payload becomes customer-visible. +2. **Given** staging/Dokploy validation is unavailable, **When** the final report is written, **Then** production data-layer readiness is no stronger than `PASS WITH CONDITIONS`. + +### Edge Cases + +- `jsonb` normalizes object key order; tests must compare decoded semantic content, not raw JSON strings. +- Duplicate JSON object keys cannot be reconstructed after `jsonb` normalization; rollback notes must record this limitation. +- Null payloads must remain null where null was the prior contract. +- Empty object/array defaults must remain unchanged where defaults already exist. +- Columns with no model, no current query usage, or unclear product ownership must not be converted silently. +- Indexes must not be added for hypothetical future dashboards, lifecycle work, or export queries. +- Sensitive payload samples must not be printed in reports, logs, screenshots, or committed fixtures. +- Changed JSON key queries must preserve workspace/managed-environment scoping before or with payload predicates. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-405-001**: Implementation MUST inventory every PostgreSQL `json` and `jsonb` column from the live schema before writing migrations. +- **FR-405-002**: The inventory matrix MUST include table, column, current type, row count, nullable state, default, model, payload purpose, existing queries, existing casts, recommended action, reason, and risk. +- **FR-405-003**: Every `json` column MUST be classified as `CONVERT`, `KEEP_JSON`, `DEPRECATED`, or `DECISION_REQUIRED`; every `jsonb` column MUST be classified as `ALREADY_JSONB`. +- **FR-405-004**: Implementation MUST convert only columns classified `CONVERT`. +- **FR-405-005**: High-risk trust-layer payloads MUST be converted or explicitly justified, including policy snapshots/metadata, backup payloads, restore preview/results/metadata, audit metadata, provider permission/readiness details, report/review/evidence payloads, alert delivery payloads, and relevant governance/finding metadata. +- **FR-405-006**: Implementation MUST preserve payload semantic content using decoded JSON equality, key-existence checks, required field checks, or domain-specific state assertions. +- **FR-405-007**: Implementation MUST preserve nullability, defaults, constraints, row counts, and application write/read behavior unless the spec is updated with an explicit exception. +- **FR-405-008**: Implementation MUST document rollback behavior and limitations, including `jsonb` key-order normalization and duplicate-key normalization. +- **FR-405-009**: Implementation MUST add indexes only for existing query paths and MUST document table/column/key, query path, reason, expected benefit, write-overhead risk, and validation proof. +- **FR-405-010**: Implementation MUST NOT add speculative GIN, expression, or partial indexes for future dashboards, lifecycle work, export work, or "might need later" reasoning. +- **FR-405-011**: Implementation MUST update model casts only where required to preserve existing Laravel behavior after conversion. +- **FR-405-012**: Implementation MUST fix code paths that assumed textual JSON representation only when tests prove the assumption breaks after `jsonb` conversion. +- **FR-405-013**: Implementation MUST not rename payload keys or change payload meaning. +- **FR-405-014**: Implementation MUST not add new product features, UI surfaces, navigation, authorization models, provider semantics, governance lifecycle behavior, report templates, or normalized replacement tables. +- **FR-405-015**: Implementation MUST prove evidence/currentness behavior from Spec 403 does not regress for converted evidence/report/review payloads. +- **FR-405-016**: Implementation MUST prove authorization and customer-safe boundaries from Specs 401 and 402 do not regress for any changed payload query path. +- **FR-405-017**: Implementation MUST prove management-report PDF/runtime behavior from Spec 404 does not regress where stored report/review payloads or report metadata are in scope. +- **FR-405-018**: Implementation MUST run PostgreSQL-specific migration/type assertions for converted columns. +- **FR-405-019**: Implementation MUST run targeted read/write tests for converted payloads. +- **FR-405-020**: Implementation MUST run focused browser proof for representative payload-backed surfaces. +- **FR-405-021**: Implementation MUST produce `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md` using the required report structure. +- **FR-405-022**: Implementation MUST record final gate result as `PASS`, `PASS WITH CONDITIONS`, or `FAIL`. +- **FR-405-023**: Implementation MUST not claim production data-layer readiness if staging-like PostgreSQL validation is unavailable or incomplete. +- **FR-405-024**: Implementation MUST preserve completed historical spec artifacts as read-only context. + +### Non-Functional Requirements + +- **NFR-405-001**: Migrations MUST be safe, incremental, reversible where practical, and explicit about lock/runtime risk. +- **NFR-405-002**: PostgreSQL-specific behavior MUST be validated in PostgreSQL, not SQLite. +- **NFR-405-003**: Reports, logs, screenshots, and test output MUST avoid secrets, tokens, raw credential payloads, and sensitive raw provider/customer payloads. +- **NFR-405-004**: The implementation MUST avoid widening test fixtures, browser setup, workspace/member context, or provider setup beyond the narrow proof needs. +- **NFR-405-005**: Any remaining P0/P1 data-layer proof gap MUST be reported, not silently deferred. + +## Key Entities *(include if feature involves data)* + +- **JSON/JSONB Column Inventory Row**: Spec-local report row representing a live schema column, its current type, ownership, usage, action, reason, and risk. +- **Conversion Migration**: One or more Laravel migrations that alter selected existing columns from `json` to `jsonb` using PostgreSQL-safe conversion. +- **Index Justification Row**: Spec-local report row proving a new JSONB index is tied to an existing query path and bounded overhead. +- **Data Validation Row**: Spec-local report row proving row counts, null counts, type changes, and semantic payload checks before and after conversion. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-405-001**: 100% of live PostgreSQL `json` and `jsonb` columns appear in the inventory matrix. +- **SC-405-002**: 100% of `json` columns have an explicit action and reason. +- **SC-405-003**: 100% of converted columns have type assertions proving `jsonb` after migration. +- **SC-405-004**: 100% of converted columns have semantic preservation proof for representative non-sensitive sample payloads or an explicit no-row/no-sample note. +- **SC-405-005**: 100% of added JSONB indexes have existing query-path justification and validation proof. +- **SC-405-006**: Targeted evidence, OperationRun, provider, backup, restore, review/report, authorization/scope, and customer-safe regression tests pass or produce documented bounded findings. +- **SC-405-007**: Focused browser proof covers at least five representative payload-backed surfaces or documents exact unavailable paths and resulting gate condition. +- **SC-405-008**: Final implementation report records gate result, validation commands, staging-like validation status, remaining findings, and recommended next step. + +## Initial Repo Truth Snapshot + +Live PostgreSQL schema inspection on 2026-06-23 found these active `json` columns requiring classification: + +```text +alert_deliveries.payload +alert_rules.tenant_allowlist +audit_logs.metadata +backup_items.assignments +backup_items.metadata +backup_items.payload +backup_schedules.days_of_week +backup_schedules.policy_types +backup_sets.metadata +managed_environment_onboarding_sessions.state +managed_environment_permissions.details +managed_environments.metadata +managed_environments.rbac_canary_results +managed_environments.rbac_last_warnings +policies.metadata +policy_versions.assignments +policy_versions.metadata +policy_versions.scope_tags +policy_versions.secret_fingerprints +policy_versions.snapshot +restore_runs.group_mapping +restore_runs.metadata +restore_runs.preview +restore_runs.requested_items +restore_runs.results +tenant_settings.value +workspace_settings.value +``` + +The same inspection found several newer trust-layer columns already using `jsonb`, including baseline, evidence, finding, review pack, review publication resolution, provider connection, stored report, support request, notification, and operation run payload columns. Implementation must still verify casts and query paths for `ALREADY_JSONB` rows where relevant. + +## Required Final Report Structure + +Implementation MUST create `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md` with these sections: + +1. Candidate Gate Result. +2. Scope Confirmation. +3. Dirty State. +4. JSON/JSONB Inventory Matrix. +5. Migrations Added. +6. Index Justification. +7. Runtime Changes Made. +8. Tests Added or Updated. +9. Data Validation. +10. Browser Proof. +11. Regression Proof. +12. Staging Validation. +13. Remaining Findings. +14. Deferred Items. +15. Validation Commands. +16. Product Surface, Filament, Deployment, and Recommended Next Step Close-Out, including tests/browser result, Livewire v4 compliance, provider registration location, global search posture, destructive/high-impact action posture, asset strategy, deployment impact, visible complexity outcome, no completed-spec rewrite assertion, and recommended next step. + +## Gate Rules + +- **PASS**: no P0/P1 findings remain; high-risk `json` columns are converted or justified; local and staging-like PostgreSQL validation pass; payload semantic preservation, targeted tests, and representative browser proof pass. +- **PASS WITH CONDITIONS**: no P0 remains; local/test migration proof is strong; remaining P1 conditions such as staging validation or rollback proof are bounded and documented. +- **FAIL**: any P0 remains; payload data is lost; migration cannot safely run; application behavior regresses; evidence/currentness or authorization/customer-safe boundaries regress; or high-risk `json` columns remain unconverted without justification. + +## Follow-up Spec Candidates + +- Governance Artifact Lifecycle & Retention runtime, only after data-layer readiness is `PASS` or conditions do not affect lifecycle storage. +- Bounded online migration strategy if any large table cannot safely use direct `ALTER COLUMN ... TYPE jsonb`. +- Provider readiness/onboarding productization if browser/regression proof reveals provider-facing user friction unrelated to storage type. +- Full browser/runtime audit remains separate and must not be folded into this spec. + +## Assumptions + +- PostgreSQL is the authoritative local/staging database for this proof. +- The product is still pre-production under the constitution, so compatibility shims are not required unless explicitly approved. +- Existing payload keys and product meanings are correct unless tests reveal a defect; this spec does not reinterpret payload schemas. +- Staging/Dokploy validation may be externally blocked; if so, final readiness must be conditional. + +## Open Questions + +- None blocking preparation. Implementation must decide column actions from live schema inventory and repo usage evidence, not from this preparation draft alone. diff --git a/specs/405-json-to-jsonb-data-layer-hardening/tasks.md b/specs/405-json-to-jsonb-data-layer-hardening/tasks.md new file mode 100644 index 00000000..a22f9fcc --- /dev/null +++ b/specs/405-json-to-jsonb-data-layer-hardening/tasks.md @@ -0,0 +1,157 @@ +# Tasks: Spec 405 - JSON-to-JSONB Data-layer Hardening + +**Input**: `specs/405-json-to-jsonb-data-layer-hardening/spec.md`, `plan.md`, `checklists/requirements.md`, user-provided Spec 405 draft, Spec 400 P1 data-layer risk, related Specs 401-404 proof context, live PostgreSQL schema inspection, and repo truth. + +**Tests**: Required. This is a runtime data-layer change. Use Pest 4, PostgreSQL lane for JSONB/migration/index proof, focused feature tests for domain behavior, and focused browser proof for existing payload-backed rendered surfaces. + +## Test Governance Checklist + +- [x] Lane assignment is `PostgreSQL + Feature + focused Browser`, with no full browser/runtime audit claim. +- [x] New or changed tests stay in the smallest honest family; browser coverage is explicit and focused. +- [x] Shared helpers, factories, seeds, fixtures, workspace/member context, and provider setup stay cheap by default or explicitly opt in. +- [x] Planned validation commands cover the change without broad unrelated suite cost. +- [x] Browser proof is required for representative existing payload-backed surfaces even though no UI file changes. +- [x] Human Product Sanity confirms unchanged trust semantics and no raw payload exposure. +- [x] Any material budget, baseline, trend, or escalation note is recorded in `implementation-report.md`. + +## Phase 1: Preparation And Safety + +**Purpose**: Establish repo safety, read the active package, and prevent scope drift. + +- [x] T001 Read `specs/405-json-to-jsonb-data-layer-hardening/spec.md`, `plan.md`, `tasks.md`, and `checklists/requirements.md`. +- [x] T002 Record current branch, HEAD, dirty state, tracked changed files, untracked files, and `git diff --check` in `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`. +- [x] T003 Re-read `AGENTS.md`, `.specify/memory/constitution.md`, `docs/ai-coding-rules.md`, `docs/architecture-guidelines.md`, `docs/performance-guidelines.md`, `docs/security-guidelines.md`, `docs/testing-guidelines.md`, and `docs/product/standards/product-surface-contract.md`. +- [x] T004 Re-read Specs 400, 401, 402, 403, and 404 implementation reports as read-only context and record their gate results/conditions in `implementation-report.md`. +- [x] T005 Confirm no UI surfaces, routes, navigation, product concepts, authorization model, lifecycle semantics, provider semantics, normalized replacement tables, or completed specs will be edited. +- [x] T006 Create `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md` with sections A through P from `spec.md`. + +## Phase 2: Schema Inventory And Usage Mapping + +**Purpose**: Prove every JSON column decision is intentional before migration work begins. + +- [x] T007 Query PostgreSQL `information_schema.columns` for all live `json` and `jsonb` columns and add each row to the JSON/JSONB Inventory Matrix in `implementation-report.md`. +- [x] T008 For every inventory row, add row count, null count, nullable state, default, indexes, constraints, foreign-key context where relevant, table-size signal, lock/runtime-risk assessment, and direct-vs-online migration decision. +- [x] T009 [P] Map model casts and model ownership for alert, audit, backup, restore, policy, provider/environment, settings, review/report, evidence, finding, and OperationRun-related rows in `apps/platform/app/Models/`. +- [x] T010 [P] Map factory and fixture usage for inventoried columns under `apps/platform/database/factories/` and `apps/platform/tests/`. +- [x] T011 [P] Map query usage for JSON paths, `whereJson*`, `->` key predicates, raw SQL JSON expressions, and JSONB index candidates under `apps/platform/app/` and `apps/platform/tests/`. +- [x] T012 [P] Map rendered/regression consumers for evidence, OperationRun, provider, backup, restore, review pack, stored report, audit, and alert payloads under `apps/platform/app/Filament/` and `apps/platform/resources/views/`. +- [x] T013 Classify each inventory row as `CONVERT`, `KEEP_JSON`, `ALREADY_JSONB`, `DEPRECATED`, or `DECISION_REQUIRED` with risk `P0`, `P1`, `P2`, `P3`, or `None`. +- [x] T014 Stop before migrations if any live `json` column lacks classification, reason, and risk. + +## Phase 3: User Story 1 - Inventory and classify structured payload columns (Priority: P1) + +**Goal**: Deliver a complete schema and usage matrix that lets reviewers verify every storage decision. + +**Independent Test**: The inventory matrix includes every live PostgreSQL `json` and `jsonb` column exactly once and every `json` column has an explicit action/reason/risk. + +### Tests for User Story 1 + +- [x] T015 [P] [US1] Add a PostgreSQL schema inventory assertion for live JSON/JSONB columns in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`. +- [x] T016 [P] [US1] Add a report-coverage assertion or manual checklist entry ensuring every live `json` column is represented in `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`. + +### Implementation for User Story 1 + +- [x] T017 [US1] Complete implementation report section D with the JSON/JSONB Inventory Matrix. +- [x] T018 [US1] Mark high-risk columns for policy snapshots, backup payloads, restore previews/results, audit metadata, provider permission/readiness details, review/report payloads, and alert delivery payloads as `CONVERT` or explicitly justified non-conversions. +- [x] T019 [US1] Mark unclear ownership/product-decision columns as `DECISION_REQUIRED` instead of converting by assumption. +- [x] T020 [US1] Record existing `ALREADY_JSONB` trust-layer columns so implementation reviewers can verify consistency and avoid duplicate work. + +## Phase 4: User Story 2 - Convert selected trust-layer payloads safely (Priority: P1) + +**Goal**: Convert selected existing `json` payload columns to `jsonb` while preserving semantic payload content and behavior. + +**Independent Test**: Migrations run on PostgreSQL, selected columns become `jsonb`, semantic payload checks pass, null/default behavior is preserved, and rollback limitations are documented. + +### Tests for User Story 2 + +- [x] T021 [P] [US2] Add PostgreSQL migration/type assertions for every converted column in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`. +- [x] T022 [P] [US2] Add semantic payload preservation tests with non-sensitive samples for policy, backup, restore, audit, provider/environment, settings, and alert/report payload categories in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`. +- [x] T023 [P] [US2] Add model read/write tests for converted columns whose casts or factories require explicit proof in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`. +- [x] T024 [P] [US2] Add query-path tests for any changed JSON key query or new JSONB index in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`. + +### Implementation for User Story 2 + +- [x] T025 [US2] Create one or more focused migrations under `apps/platform/database/migrations/` converting only `CONVERT` columns to `jsonb`. +- [x] T026 [US2] Preserve nullability, defaults, and constraints in each conversion migration. +- [x] T027 [US2] Add rollback SQL or explicit rollback limitations to each conversion migration. +- [x] T028 [US2] Add justified JSONB indexes only where an existing query path is documented in `implementation-report.md`. +- [x] T029 [US2] Update model casts in `apps/platform/app/Models/` only where tests prove current behavior needs a cast adjustment. +- [x] T030 [US2] Update query code only where tests prove textual JSON assumptions break after conversion. +- [x] T031 [US2] Update factories or fixtures only where required by the new targeted tests and avoid broad seed/default changes. +- [x] T032 [US2] Complete implementation report sections E, F, G, and I with migrations, indexes, runtime changes, and data validation. + +## Phase 5: User Story 3 - Prove existing behavior and report readiness honestly (Priority: P2) + +**Goal**: Prove the conversion did not regress existing trust-layer behavior and produce a gate result. + +**Independent Test**: Targeted domain tests, PostgreSQL tests, focused browser proof, and staging-like validation status are recorded; final report uses `PASS`, `PASS WITH CONDITIONS`, or `FAIL` honestly. + +### Tests for User Story 3 + +- [x] T033 [P] [US3] Prove evidence/currentness regression paths with existing focused browser smoke and Spec405 storage tests; no converted evidence/report/review columns required new feature tests. +- [x] T034 [P] [US3] Add OperationRun/audit regression proof for converted audit and OperationRun-adjacent payload paths in the focused Spec405 test plus browser proof. +- [x] T035 [P] [US3] Add provider readiness/freshness/permission regression proof for converted provider/environment payload paths in the focused Spec405 test plus browser proof. +- [x] T036 [P] [US3] Add backup payload regression proof in the focused Spec405 test plus browser proof. +- [x] T037 [P] [US3] Add restore preview/result payload regression proof in the focused Spec405 test plus browser proof. +- [x] T038 [P] [US3] Prove review pack, stored report, management-report PDF/runtime, and customer-output regression paths with existing focused browser smoke; those columns were already `jsonb`. +- [x] T039 [P] [US3] Confirm no authorization/scope query path changed beyond the JSONB-safe `BackupItem` scope; covered by targeted tests and no RBAC edits. +- [x] T040 [US3] Run focused existing browser smoke tests for representative existing payload-backed surfaces; no new UI smoke artifact was needed because no rendered surface changed. + +### Implementation for User Story 3 + +- [x] T041 [US3] Browser-proof Evidence Overview or evidence anchor rendering after conversion and record route, actor, workspace/environment, expected state, actual state, and console/runtime result. +- [x] T042 [US3] Browser-proof Monitoring -> Operations or OperationRun detail payload-backed rendering after conversion. +- [x] T043 [US3] Browser-proof provider detail/readiness/freshness payload-backed rendering after conversion. +- [x] T044 [US3] Browser-proof backup or restore payload-backed proof rendering after conversion. +- [x] T045 [US3] Browser-proof Review Pack, Stored Report, management-report PDF/runtime, or report receipt payload-backed rendering after conversion. +- [x] T046 [US3] Record Human Product Sanity result in `implementation-report.md`: no stronger proof claims, no raw payload exposure, unchanged current/released/failed/partial semantics. +- [x] T047 [US3] Run local PostgreSQL migration validation and record migration success, row counts, null counts, converted types, and sample semantic checks in `implementation-report.md`. +- [x] T048 [US3] Run rollback validation in a disposable local/test database where safe, or record exact rollback limitation and risk. +- [x] T049 [US3] Run staging-like PostgreSQL validation if accessible and record migration/app boot/tests/browser/payload-read-write results. +- [x] T050 [US3] If staging/Dokploy validation is not accessible, record the blocker and ensure final gate result is no stronger than `PASS WITH CONDITIONS`. +- [x] T051 [US3] Complete implementation report sections H, J, K, L, M, N, O, and P. + +## Phase 6: Final Validation And Close-Out + +**Purpose**: Confirm the package is ready for review and no unrelated work entered the slice. + +- [x] T052 Run `git diff --check` from repo root and record result in `implementation-report.md`. +- [x] T053 Run project formatting for changed PHP files according to repo convention and record result. +- [x] T054 Run focused PostgreSQL Spec405 tests and record exact command/results. +- [x] T055 Run targeted feature/domain regression tests and record exact command/results. +- [x] T056 Run focused browser proof and record exact command/results, or exact blocker without claiming browser proof. +- [x] T057 Verify reports, logs, screenshots, and fixtures do not include secrets, tokens, raw credential payloads, sensitive raw provider payloads, or customer-sensitive raw payloads. +- [x] T058 Record final dirty state, tracked changed files, and untracked files in `implementation-report.md`. +- [x] T059 Confirm no completed historical specs were rewritten or stripped of close-out/validation/task/smoke/browser history. +- [x] T060 Confirm `implementation-report.md` and the final response state Livewire v4 compliance, provider registration location, global search posture, destructive/high-impact action posture, asset strategy, tests/browser result, deployment impact, visible complexity outcome, no completed-spec rewrite assertion, and explicit application implementation status. + +## Non-Goals Checklist + +- [x] NT001 Do not add product features, report templates, navigation, UI surfaces, actions, modals, forms, tables, or customer output. +- [x] NT002 Do not change authorization model, roles, capabilities, policies, or global search posture except to fix a direct regression caused by this conversion and already covered by the spec. +- [x] NT003 Do not add governance lifecycle, retention, export, delete, hold, or artifact-state semantics. +- [x] NT004 Do not add new persisted entities/tables, normalized replacement tables, status families, taxonomies, provider frameworks, or broad abstractions. +- [x] NT005 Do not add speculative indexes. +- [x] NT006 Do not rename payload keys or reinterpret payload meaning. +- [x] NT007 Do not print or commit sensitive raw payload samples. +- [x] NT008 Do not rewrite completed specs or remove historical implementation evidence. + +## Dependencies And Execution Order + +- Phase 1 must complete before any migration or runtime edit. +- Phase 2 inventory and classification must complete before Phase 4 conversion. +- User Story 1 is the required MVP and blocks migration implementation. +- User Story 2 depends on completed classification and creates the conversion. +- User Story 3 depends on converted columns and proves behavior/readiness. +- Final validation depends on all in-scope tests, browser proof, and implementation report completion. + +## Parallel Execution Examples + +- After T007/T008, T009 through T012 can run in parallel because they inspect different surfaces. +- T021 through T024 can be drafted in parallel after conversion decisions are known. +- T033 through T039 can run in parallel by domain test family after migrations exist. +- T041 through T045 can run as separate focused browser paths if fixture setup is independent. + +## Recommended Implementation Strategy + +Deliver User Story 1 first and stop if the inventory is incomplete or reveals unresolved product/schema decisions. Then convert the smallest justified group of high-risk trust-layer columns, prove semantic preservation, and add only the narrow tests/browser proof required for the changed columns. Treat staging validation as a release gate: if unavailable, report `PASS WITH CONDITIONS`, not full readiness.