ahmido/TenantAtlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: harden json to jsonb data layer for trust payloads
Some checks failed
PR Fast Feedback / fast-feedback (pull_request) Failing after 1m6s
Details

Browse Source
This commit is contained in:
Ahmed Darrazi 2026-06-23 23:36:12 +02:00
parent 8918b35795
commit 439a3b4eda
8 changed files with 2078 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/platform/app/Models/BackupItem.php
View File

@ -174,7 +174,11 @@ public function resolvedDisplayName(): string
// Scopes
public function scopeWithAssignments($query)
{
$arrayLengthExpression = $query->getConnection()->getDriverName() === 'pgsql'
? 'jsonb_array_length(assignments::jsonb)'
: 'json_array_length(assignments)';
return $query->whereNotNull('assignments')
->whereRaw('json_array_length(assignments) > 0');
->whereRaw($arrayLengthExpression.' > 0');
}
}

107
apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php Normal file
View File

@ -0,0 +1,107 @@
<?php
declare(strict_types=1);
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\DB;
return new class extends Migration
{
/**
* @var list<array{0: string, 1: string}>
*/
private const CONVERTED_JSON_COLUMNS = [
['alert_deliveries', 'payload'],
['alert_rules', 'tenant_allowlist'],
['audit_logs', 'metadata'],
['backup_items', 'assignments'],
['backup_items', 'metadata'],
['backup_items', 'payload'],
['backup_schedules', 'days_of_week'],
['backup_schedules', 'policy_types'],
['backup_sets', 'metadata'],
['managed_environment_onboarding_sessions', 'state'],
['managed_environment_permissions', 'details'],
['managed_environments', 'metadata'],
['managed_environments', 'rbac_canary_results'],
['managed_environments', 'rbac_last_warnings'],
['policies', 'metadata'],
['policy_versions', 'assignments'],
['policy_versions', 'metadata'],
['policy_versions', 'scope_tags'],
['policy_versions', 'secret_fingerprints'],
['policy_versions', 'snapshot'],
['restore_runs', 'group_mapping'],
['restore_runs', 'metadata'],
['restore_runs', 'preview'],
['restore_runs', 'requested_items'],
['restore_runs', 'results'],
['tenant_settings', 'value'],
['workspace_settings', 'value'],
];
public function up(): void
{
if (DB::getDriverName() !== 'pgsql') {
return;
}
foreach ($this->columnsGroupedByTable(self::CONVERTED_JSON_COLUMNS) as $table => $columns) {
$this->alterJsonColumnTypes($table, $columns, 'jsonb');
}
}
public function down(): void
{
if (DB::getDriverName() !== 'pgsql') {
return;
}
foreach ($this->columnsGroupedByTable(array_reverse(self::CONVERTED_JSON_COLUMNS)) as $table => $columns) {
$this->alterJsonColumnTypes($table, $columns, 'json');
}
}
/**
* @param list<array{0: string, 1: string}> $columns
* @return array<string, list<string>>
*/
private function columnsGroupedByTable(array $columns): array
{
$grouped = [];
foreach ($columns as [$table, $column]) {
$grouped[$table][] = $column;
}
return $grouped;
}
/**
* @param list<string> $columns
*/
private function alterJsonColumnTypes(string $table, array $columns, string $type): void
{
$clauses = array_map(
fn (string $column): string => sprintf(
'ALTER COLUMN %s TYPE %s USING %s::%s',
$this->quoteIdentifier($column),
$type,
$this->quoteIdentifier($column),
$type,
),
$columns,
);
DB::statement(sprintf(
'ALTER TABLE %s %s',
$this->quoteIdentifier($table),
implode(', ', $clauses),
));
}
private function quoteIdentifier(string $identifier): string
{
return '"'.str_replace('"', '""', $identifier).'"';
}
};

591
apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php Normal file
View File

@ -0,0 +1,591 @@
<?php
declare(strict_types=1);
use App\Models\AlertDelivery;
use App\Models\AlertRule;
use App\Models\AuditLog;
use App\Models\BackupItem;
use App\Models\BackupSchedule;
use App\Models\BackupSet;
use App\Models\ManagedEnvironment;
use App\Models\ManagedEnvironmentOnboardingSession;
use App\Models\ManagedEnvironmentPermission;
use App\Models\Policy;
use App\Models\PolicyVersion;
use App\Models\RestoreRun;
use App\Models\TenantSetting;
use App\Models\WorkspaceSetting;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\DB;
beforeEach(function (): void {
if (DB::getDriverName() !== 'pgsql') {
$this->markTestSkipped('Spec 405 JSONB hardening is PostgreSQL-only.');
}
});
it('converts every reviewed legacy json column to jsonb', function (): void {
$columnTypes = spec405ColumnTypes();
foreach (spec405ConvertedJsonColumns() as [$table, $column]) {
$qualifiedColumn = "{$table}.{$column}";
expect($columnTypes)
->toHaveKey($qualifiedColumn)
->and($columnTypes[$qualifiedColumn])->toBe('jsonb');
}
$remainingJsonColumns = DB::table('information_schema.columns')
->where('table_schema', 'public')
->where('data_type', 'json')
->orderBy('table_name')
->orderBy('column_name')
->get(['table_name', 'column_name'])
->map(fn (object $column): string => "{$column->table_name}.{$column->column_name}")
->all();
expect($remainingJsonColumns)->toBeEmpty();
});
it('does not introduce speculative gin indexes for converted payload columns', function (): void {
$convertedTables = collect(spec405ConvertedJsonColumns())
->pluck(0)
->unique()
->values()
->all();
$convertedGinIndexes = DB::table('pg_indexes')
->where('schemaname', 'public')
->whereIn('tablename', $convertedTables)
->where('indexdef', 'ilike', '%USING gin%')
->orderBy('tablename')
->orderBy('indexname')
->get(['tablename', 'indexname'])
->map(fn (object $index): string => "{$index->tablename}.{$index->indexname}")
->all();
expect($convertedGinIndexes)->toBeEmpty();
});
it('preserves existing json rows attributes constraints indexes and rollback query compatibility', function (): void {
$migration = spec405Migration();
$columnContractsBefore = spec405ColumnContracts();
$indexDefinitionsBefore = spec405IndexDefinitionsForConvertedTables();
$constraintDefinitionsBefore = spec405ConstraintDefinitionsForConvertedTables();
$migration->down();
try {
$jsonColumnContracts = spec405ColumnContracts();
foreach (spec405ConvertedJsonColumns() as [$table, $column]) {
$qualifiedColumn = "{$table}.{$column}";
expect($jsonColumnContracts[$qualifiedColumn]['data_type'])->toBe('json')
->and($jsonColumnContracts[$qualifiedColumn]['nullable'])->toBe($columnContractsBefore[$qualifiedColumn]['nullable'])
->and($jsonColumnContracts[$qualifiedColumn]['default'])->toBe($columnContractsBefore[$qualifiedColumn]['default']);
}
$ids = spec405CreateRepresentativePayloadRows();
$payloadsBefore = spec405RepresentativePayloads($ids);
expect(BackupItem::withAssignments()->pluck('id')->map(fn (int $id): int => $id)->all())
->toContain($ids['backup_item_id']);
$migration->up();
expect(spec405ColumnContracts())->toBe($columnContractsBefore)
->and(spec405IndexDefinitionsForConvertedTables())->toBe($indexDefinitionsBefore)
->and(spec405ConstraintDefinitionsForConvertedTables())->toBe($constraintDefinitionsBefore)
->and(spec405RepresentativePayloads($ids))->toMatchArray($payloadsBefore);
expect(BackupItem::withAssignments()->pluck('id')->map(fn (int $id): int => $id)->all())
->toContain($ids['backup_item_id']);
$migration->down();
expect(spec405ColumnTypes()['backup_items.assignments'])->toBe('json')
->and(BackupItem::withAssignments()->pluck('id')->map(fn (int $id): int => $id)->all())
->toContain($ids['backup_item_id']);
} finally {
if ((spec405ColumnTypes()['backup_items.assignments'] ?? null) !== 'jsonb') {
$migration->up();
}
}
});
it('preserves representative jsonb read write and cast behavior', function (): void {
$tenant = ManagedEnvironment::factory()->create([
'metadata' => ['source' => 'spec405', 'nested' => ['enabled' => true]],
]);
DB::table('managed_environments')
->where('id', (int) $tenant->getKey())
->update([
'rbac_canary_results' => json_encode(['status' => 'pass', 'checks' => ['read' => true]], JSON_THROW_ON_ERROR),
'rbac_last_warnings' => json_encode(['warnings' => ['scope review required']], JSON_THROW_ON_ERROR),
]);
$tenant->refresh();
expect($tenant->metadata)->toMatchArray(['nested' => ['enabled' => true], 'source' => 'spec405'])
->and($tenant->rbac_canary_results)->toMatchArray(['checks' => ['read' => true], 'status' => 'pass'])
->and($tenant->rbac_last_warnings)->toMatchArray(['warnings' => ['scope review required']]);
$backupSet = BackupSet::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'metadata' => ['source' => 'spec405'],
]);
$policy = Policy::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'metadata' => ['platform_family' => 'windows'],
]);
$backupItem = BackupItem::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'backup_set_id' => (int) $backupSet->getKey(),
'policy_id' => (int) $policy->getKey(),
'payload' => ['id' => 'policy-jsonb', 'settings' => [['name' => 'mode', 'value' => 'audit']]],
'metadata' => ['scope_tag_names' => ['Default', 'Security']],
'assignments' => [['id' => 'assignment-1', 'target' => ['groupId' => 'group-jsonb']]],
]);
expect($backupItem->refresh()->payload)->toMatchArray(['id' => 'policy-jsonb', 'settings' => [['name' => 'mode', 'value' => 'audit']]])
->and($backupItem->metadata)->toMatchArray(['scope_tag_names' => ['Default', 'Security']])
->and($backupItem->assignments)->toBe([['id' => 'assignment-1', 'target' => ['groupId' => 'group-jsonb']]]);
$policyVersion = PolicyVersion::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'policy_id' => (int) $policy->getKey(),
'snapshot' => ['displayName' => 'Spec 405 policy', 'settings' => [['id' => 'setting-1']]],
'metadata' => ['source' => 'unit'],
'assignments' => [['target' => ['groupId' => 'group-jsonb']]],
'scope_tags' => ['0', 'security'],
'secret_fingerprints' => ['snapshot' => [], 'assignments' => ['hash']],
]);
expect($policyVersion->refresh()->snapshot)->toMatchArray(['displayName' => 'Spec 405 policy', 'settings' => [['id' => 'setting-1']]])
->and($policyVersion->metadata)->toMatchArray(['source' => 'unit'])
->and($policyVersion->assignments)->toBe([['target' => ['groupId' => 'group-jsonb']]])
->and($policyVersion->scope_tags)->toBe(['0', 'security'])
->and($policyVersion->secret_fingerprints)->toMatchArray(['assignments' => ['hash'], 'snapshot' => []]);
$restoreRun = RestoreRun::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'backup_set_id' => (int) $backupSet->getKey(),
'requested_items' => [['policy_identifier' => 'policy-jsonb']],
'preview' => [['action' => 'update', 'policy_identifier' => 'policy-jsonb']],
'results' => ['items' => [['status' => 'applied']]],
'metadata' => ['dry_run' => false],
'group_mapping' => ['old-group' => 'new-group'],
]);
expect($restoreRun->refresh()->requested_items)->toBe([['policy_identifier' => 'policy-jsonb']])
->and($restoreRun->preview)->toBe([['action' => 'update', 'policy_identifier' => 'policy-jsonb']])
->and($restoreRun->results)->toMatchArray(['items' => [['status' => 'applied']]])
->and($restoreRun->metadata)->toMatchArray(['dry_run' => false])
->and($restoreRun->group_mapping)->toMatchArray(['old-group' => 'new-group']);
$auditLog = AuditLog::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'action' => 'spec405.jsonb.regression',
'resource_type' => 'policy',
'resource_id' => 'policy-jsonb',
'status' => 'success',
'metadata' => ['_dedupe_key' => 'spec405-jsonb', 'reason' => 'schema regression'],
'recorded_at' => now(),
]);
expect($auditLog->refresh()->metadata)->toHaveKey('_dedupe_key', 'spec405-jsonb');
$metadataLookupExists = AuditLog::query()
->whereRaw("metadata ->> '_dedupe_key' = ?", ['spec405-jsonb'])
->exists();
expect($metadataLookupExists)->toBeTrue();
$schedule = BackupSchedule::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'name' => 'Spec 405 weekly backup',
'frequency' => 'weekly',
'time_of_day' => '03:00:00',
'days_of_week' => [1, 3, 5],
'policy_types' => ['settingsCatalogPolicy', 'deviceConfiguration'],
]);
expect($schedule->refresh()->days_of_week)->toBe([1, 3, 5])
->and($schedule->policy_types)->toBe(['settingsCatalogPolicy', 'deviceConfiguration']);
$alertRule = AlertRule::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'tenant_allowlist' => [(int) $tenant->getKey()],
]);
$alertDelivery = AlertDelivery::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'alert_rule_id' => (int) $alertRule->getKey(),
'payload' => ['title' => 'Spec 405', 'context' => ['severity' => 'high']],
]);
$permission = ManagedEnvironmentPermission::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'permission_key' => 'DeviceManagementConfiguration.Read.All',
'status' => 'granted',
'details' => ['source' => 'graph', 'roles' => ['reader']],
'last_checked_at' => now(),
]);
$onboardingSession = ManagedEnvironmentOnboardingSession::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'state' => ['entra_tenant_id' => 'tenant-jsonb', 'step' => 'verify'],
]);
$tenantSetting = TenantSetting::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'value' => ['keep_last' => 45],
]);
$workspaceSetting = WorkspaceSetting::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'value' => ['keep_last' => 30],
]);
expect($backupSet->refresh()->metadata)->toMatchArray(['source' => 'spec405'])
->and($policy->refresh()->metadata)->toMatchArray(['platform_family' => 'windows'])
->and($alertRule->refresh()->tenant_allowlist)->toBe([(int) $tenant->getKey()])
->and($alertDelivery->refresh()->payload)->toMatchArray(['context' => ['severity' => 'high'], 'title' => 'Spec 405'])
->and($permission->refresh()->details)->toMatchArray(['roles' => ['reader'], 'source' => 'graph'])
->and($onboardingSession->refresh()->state)->toMatchArray(['entra_tenant_id' => 'tenant-jsonb'])
->and($tenantSetting->refresh()->value)->toMatchArray(['keep_last' => 45])
->and($workspaceSetting->refresh()->value)->toMatchArray(['keep_last' => 30]);
});
it('keeps backup item assignment filtering compatible with jsonb columns', function (): void {
BackupItem::factory()->create(['assignments' => null]);
BackupItem::factory()->create(['assignments' => []]);
$withAssignments = BackupItem::factory()->create([
'assignments' => [
['id' => 'assignment-1', 'target' => ['groupId' => 'group-1']],
],
]);
$resultIds = BackupItem::withAssignments()
->pluck('id')
->map(fn (int $id): int => (int) $id)
->all();
expect($resultIds)->toBe([(int) $withAssignments->getKey()]);
});
/**
* @return list<array{0: string, 1: string}>
*/
function spec405ConvertedJsonColumns(): array
{
return [
['alert_deliveries', 'payload'],
['alert_rules', 'tenant_allowlist'],
['audit_logs', 'metadata'],
['backup_items', 'assignments'],
['backup_items', 'metadata'],
['backup_items', 'payload'],
['backup_schedules', 'days_of_week'],
['backup_schedules', 'policy_types'],
['backup_sets', 'metadata'],
['managed_environment_onboarding_sessions', 'state'],
['managed_environment_permissions', 'details'],
['managed_environments', 'metadata'],
['managed_environments', 'rbac_canary_results'],
['managed_environments', 'rbac_last_warnings'],
['policies', 'metadata'],
['policy_versions', 'assignments'],
['policy_versions', 'metadata'],
['policy_versions', 'scope_tags'],
['policy_versions', 'secret_fingerprints'],
['policy_versions', 'snapshot'],
['restore_runs', 'group_mapping'],
['restore_runs', 'metadata'],
['restore_runs', 'preview'],
['restore_runs', 'requested_items'],
['restore_runs', 'results'],
['tenant_settings', 'value'],
['workspace_settings', 'value'],
];
}
function spec405Migration(): Migration
{
return require database_path('migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php');
}
/**
* @return list<string>
*/
function spec405ConvertedTables(): array
{
return collect(spec405ConvertedJsonColumns())
->pluck(0)
->unique()
->values()
->all();
}
/**
* @return array<string, array{data_type: string, nullable: string, default: ?string}>
*/
function spec405ColumnContracts(): array
{
return DB::table('information_schema.columns')
->where('table_schema', 'public')
->whereIn('table_name', spec405ConvertedTables())
->orderBy('table_name')
->orderBy('column_name')
->get(['table_name', 'column_name', 'data_type', 'is_nullable', 'column_default'])
->mapWithKeys(fn (object $column): array => [
"{$column->table_name}.{$column->column_name}" => [
'data_type' => (string) $column->data_type,
'nullable' => (string) $column->is_nullable,
'default' => $column->column_default === null ? null : (string) $column->column_default,
],
])
->only(collect(spec405ConvertedJsonColumns())->map(fn (array $column): string => "{$column[0]}.{$column[1]}")->all())
->all();
}
/**
* @return array<string, string>
*/
function spec405IndexDefinitionsForConvertedTables(): array
{
return DB::table('pg_indexes')
->where('schemaname', 'public')
->whereIn('tablename', spec405ConvertedTables())
->orderBy('tablename')
->orderBy('indexname')
->get(['tablename', 'indexname', 'indexdef'])
->mapWithKeys(fn (object $index): array => [
"{$index->tablename}.{$index->indexname}" => (string) $index->indexdef,
])
->all();
}
/**
* @return array<string, string>
*/
function spec405ConstraintDefinitionsForConvertedTables(): array
{
return DB::table('pg_constraint')
->selectRaw('conrelid::regclass::text as table_name, conname, pg_get_constraintdef(oid) as definition')
->whereRaw("connamespace = 'public'::regnamespace")
->whereIn(DB::raw('conrelid::regclass::text'), spec405ConvertedTables())
->orderBy('table_name')
->orderBy('conname')
->get()
->mapWithKeys(fn (object $constraint): array => [
"{$constraint->table_name}.{$constraint->conname}" => (string) $constraint->definition,
])
->all();
}
/**
* @return array<string, int>
*/
function spec405CreateRepresentativePayloadRows(): array
{
$tenant = ManagedEnvironment::factory()->create([
'metadata' => ['source' => 'spec405', 'nested' => ['enabled' => true]],
]);
DB::table('managed_environments')
->where('id', (int) $tenant->getKey())
->update([
'rbac_canary_results' => json_encode(['status' => 'pass', 'checks' => ['read' => true]], JSON_THROW_ON_ERROR),
'rbac_last_warnings' => json_encode(['warnings' => ['scope review required']], JSON_THROW_ON_ERROR),
]);
$backupSet = BackupSet::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'metadata' => ['source' => 'spec405'],
]);
$policy = Policy::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'metadata' => ['platform_family' => 'windows'],
]);
$backupItem = BackupItem::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'backup_set_id' => (int) $backupSet->getKey(),
'policy_id' => (int) $policy->getKey(),
'payload' => ['id' => 'policy-jsonb', 'settings' => [['name' => 'mode', 'value' => 'audit']]],
'metadata' => ['scope_tag_names' => ['Default', 'Security']],
'assignments' => [['id' => 'assignment-1', 'target' => ['groupId' => 'group-jsonb']]],
]);
$policyVersion = PolicyVersion::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'policy_id' => (int) $policy->getKey(),
'snapshot' => ['displayName' => 'Spec 405 policy', 'settings' => [['id' => 'setting-1']]],
'metadata' => ['source' => 'unit'],
'assignments' => [['target' => ['groupId' => 'group-jsonb']]],
'scope_tags' => ['0', 'security'],
'secret_fingerprints' => ['snapshot' => [], 'assignments' => ['hash']],
]);
$restoreRun = RestoreRun::factory()->create([
'managed_environment_id' => (int) $tenant->getKey(),
'workspace_id' => (int) $tenant->workspace_id,
'backup_set_id' => (int) $backupSet->getKey(),
'requested_items' => [['policy_identifier' => 'policy-jsonb']],
'preview' => [['action' => 'update', 'policy_identifier' => 'policy-jsonb']],
'results' => ['items' => [['status' => 'applied']]],
'metadata' => ['dry_run' => false],
'group_mapping' => ['old-group' => 'new-group'],
]);
$auditLog = AuditLog::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'action' => 'spec405.jsonb.regression',
'resource_type' => 'policy',
'resource_id' => 'policy-jsonb',
'status' => 'success',
'metadata' => ['_dedupe_key' => 'spec405-jsonb', 'reason' => 'schema regression'],
'recorded_at' => now(),
]);
$schedule = BackupSchedule::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'name' => 'Spec 405 weekly backup',
'frequency' => 'weekly',
'time_of_day' => '03:00:00',
'days_of_week' => [1, 3, 5],
'policy_types' => ['settingsCatalogPolicy', 'deviceConfiguration'],
]);
$alertRule = AlertRule::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'tenant_allowlist' => [(int) $tenant->getKey()],
]);
$alertDelivery = AlertDelivery::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'alert_rule_id' => (int) $alertRule->getKey(),
'payload' => ['title' => 'Spec 405', 'context' => ['severity' => 'high']],
]);
$permission = ManagedEnvironmentPermission::query()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'permission_key' => 'DeviceManagementConfiguration.Read.All',
'status' => 'granted',
'details' => ['source' => 'graph', 'roles' => ['reader']],
'last_checked_at' => now(),
]);
$onboardingSession = ManagedEnvironmentOnboardingSession::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'state' => ['entra_tenant_id' => 'tenant-jsonb', 'step' => 'verify'],
]);
$tenantSetting = TenantSetting::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'managed_environment_id' => (int) $tenant->getKey(),
'value' => ['keep_last' => 45],
]);
$workspaceSetting = WorkspaceSetting::factory()->create([
'workspace_id' => (int) $tenant->workspace_id,
'value' => ['keep_last' => 30],
]);
return [
'alert_delivery_id' => (int) $alertDelivery->getKey(),
'alert_rule_id' => (int) $alertRule->getKey(),
'audit_log_id' => (int) $auditLog->getKey(),
'backup_item_id' => (int) $backupItem->getKey(),
'backup_schedule_id' => (int) $schedule->getKey(),
'backup_set_id' => (int) $backupSet->getKey(),
'managed_environment_id' => (int) $tenant->getKey(),
'managed_environment_onboarding_session_id' => (int) $onboardingSession->getKey(),
'managed_environment_permission_id' => (int) $permission->getKey(),
'policy_id' => (int) $policy->getKey(),
'policy_version_id' => (int) $policyVersion->getKey(),
'restore_run_id' => (int) $restoreRun->getKey(),
'tenant_setting_id' => (int) $tenantSetting->getKey(),
'workspace_setting_id' => (int) $workspaceSetting->getKey(),
];
}
/**
* @param array<string, int> $ids
* @return array<string, mixed>
*/
function spec405RepresentativePayloads(array $ids): array
{
$alertDelivery = AlertDelivery::query()->findOrFail($ids['alert_delivery_id']);
$alertRule = AlertRule::query()->findOrFail($ids['alert_rule_id']);
$auditLog = AuditLog::query()->findOrFail($ids['audit_log_id']);
$backupItem = BackupItem::query()->findOrFail($ids['backup_item_id']);
$backupSchedule = BackupSchedule::query()->findOrFail($ids['backup_schedule_id']);
$backupSet = BackupSet::query()->findOrFail($ids['backup_set_id']);
$tenant = ManagedEnvironment::query()->findOrFail($ids['managed_environment_id']);
$onboardingSession = ManagedEnvironmentOnboardingSession::query()->findOrFail($ids['managed_environment_onboarding_session_id']);
$permission = ManagedEnvironmentPermission::query()->findOrFail($ids['managed_environment_permission_id']);
$policy = Policy::query()->findOrFail($ids['policy_id']);
$policyVersion = PolicyVersion::query()->findOrFail($ids['policy_version_id']);
$restoreRun = RestoreRun::query()->findOrFail($ids['restore_run_id']);
$tenantSetting = TenantSetting::query()->findOrFail($ids['tenant_setting_id']);
$workspaceSetting = WorkspaceSetting::query()->findOrFail($ids['workspace_setting_id']);
return [
'alert_deliveries.payload' => $alertDelivery->payload,
'alert_rules.tenant_allowlist' => $alertRule->tenant_allowlist,
'audit_logs.metadata' => $auditLog->metadata,
'backup_items.assignments' => $backupItem->assignments,
'backup_items.metadata' => $backupItem->metadata,
'backup_items.payload' => $backupItem->payload,
'backup_schedules.days_of_week' => $backupSchedule->days_of_week,
'backup_schedules.policy_types' => $backupSchedule->policy_types,
'backup_sets.metadata' => $backupSet->metadata,
'managed_environment_onboarding_sessions.state' => $onboardingSession->state,
'managed_environment_permissions.details' => $permission->details,
'managed_environments.metadata' => $tenant->metadata,
'managed_environments.rbac_canary_results' => $tenant->rbac_canary_results,
'managed_environments.rbac_last_warnings' => $tenant->rbac_last_warnings,
'policies.metadata' => $policy->metadata,
'policy_versions.assignments' => $policyVersion->assignments,
'policy_versions.metadata' => $policyVersion->metadata,
'policy_versions.scope_tags' => $policyVersion->scope_tags,
'policy_versions.secret_fingerprints' => $policyVersion->secret_fingerprints,
'policy_versions.snapshot' => $policyVersion->snapshot,
'restore_runs.group_mapping' => $restoreRun->group_mapping,
'restore_runs.metadata' => $restoreRun->metadata,
'restore_runs.preview' => $restoreRun->preview,
'restore_runs.requested_items' => $restoreRun->requested_items,
'restore_runs.results' => $restoreRun->results,
'tenant_settings.value' => $tenantSetting->value,
'workspace_settings.value' => $workspaceSetting->value,
];
}
/**
* @return array<string, string>
*/
function spec405ColumnTypes(): array
{
$tables = collect(spec405ConvertedJsonColumns())
->pluck(0)
->unique()
->values()
->all();
return DB::table('information_schema.columns')
->where('table_schema', 'public')
->whereIn('table_name', $tables)
->get(['table_name', 'column_name', 'data_type'])
->mapWithKeys(fn (object $column): array => [
"{$column->table_name}.{$column->column_name}" => (string) $column->data_type,
])
->all();
}

45
specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md Normal file
View File

@ -0,0 +1,45 @@
# Specification Quality Checklist: Spec 405 - JSON-to-JSONB Data-layer Hardening
**Purpose**: Validate specification completeness and quality before implementation preparation is handed off.
**Created**: 2026-06-23
**Feature**: `specs/405-json-to-jsonb-data-layer-hardening/spec.md`
## Content Quality
- [x] Focused on product value, operator trust, and data-layer hardening risk.
- [x] No application implementation is performed by this preparation package.
- [x] Runtime implementation details are limited to expected migration/test/validation surfaces.
- [x] Mandatory Spec Kit sections are completed or explicitly marked N/A with rationale.
- [x] Scope excludes new product behavior, UI surfaces, authorization model changes, lifecycle semantics, and broad abstractions.
## Requirement Completeness
- [x] No unresolved clarification markers remain.
- [x] Requirements are testable and unambiguous.
- [x] Success criteria are measurable.
- [x] Acceptance scenarios are defined for inventory, conversion, and regression/reporting.
- [x] Edge cases are identified, including `jsonb` key order normalization and rollback limitations.
- [x] Dependencies and assumptions are identified.
- [x] Required final implementation report structure is defined.
## Feature Readiness
- [x] Functional requirements map to concrete tasks in `tasks.md`.
- [x] User stories cover the minimum viable implementation sequence.
- [x] The plan identifies likely affected repository surfaces without authorizing unrelated runtime edits.
- [x] Tasks include tests, PostgreSQL validation, focused browser proof, staging-like validation handling, and final close-out.
## Constitution And Product Surface Readiness
- [x] Spec Candidate Check is completed with approval class, score, and decision.
- [x] Completed-spec guardrail is explicit and related specs are read-only context.
- [x] No UI surface impact is checked with a clear rationale.
- [x] Product Surface Contract is handled as no-rendered-surface-change plus focused regression proof.
- [x] Browser verification is required as backend regression proof for existing payload-backed surfaces.
- [x] Human Product Sanity is scoped to unchanged trust semantics.
- [x] Proportionality review states no new runtime framework, persisted entity, status family, or UI taxonomy.
- [x] Test governance names PostgreSQL, feature, and focused browser lanes with fixture-cost controls.
## Notes
Preparation review result: pass. The package is ready for a separate implementation loop, provided the implementation agent completes the inventory matrix before writing migrations and preserves the no-product/no-UI boundary.

331
specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md Normal file
View File

@ -0,0 +1,331 @@
# Spec 405 Implementation Report - JSON-to-JSONB Data-layer Hardening
## 1. Candidate Gate Result
**Current result**: `PASS WITH CONDITIONS`.
Local PostgreSQL migration/type proof, rollback/down-up proof, schema-attribute proof, and focused model/query regression tests pass. The remaining condition is external staging/Dokploy validation; it is not accessible from this agent session, so the gate cannot honestly be stronger than `PASS WITH CONDITIONS`.
## 2. Scope Confirmation
In scope:
- Inventory every live PostgreSQL `json` and `jsonb` column.
- Classify every live `json` column.
- Convert only reviewed `CONVERT` columns to `jsonb`.
- Preserve current payload semantics, casts, scoping, and query behavior.
- Add local PostgreSQL tests and focused regression proof.
Out of scope and not changed:
- UI surfaces, routes, navigation, Filament resources, panels, actions, forms, tables, widgets, or customer output.
- Authorization model, roles, capabilities, provider semantics, lifecycle semantics, normalized replacement tables, and new product concepts.
- Completed historical specs and their validation/task/smoke/browser history.
- Speculative JSONB indexes.
Historical context reviewed as read-only:
- Spec 400: no `implementation-report.md` present; spec/plan/tasks were treated as historical audit context only.
- Spec 401: backup confirmation and provider residual context reviewed; left unchanged.
- Spec 402: provider action residual closure context reviewed; left unchanged.
- Spec 403: evidence/currentness runtime closure context reviewed; left unchanged.
- Spec 404: management report PDF staging validation and `PASS WITH CONDITIONS` staging caveat reviewed; left unchanged.
## 3. Dirty State
Initial repository state:
- Branch: `405-json-to-jsonb-data-layer-hardening`
- HEAD: `8918b357 feat: finish management report PDF staging validation (#475)`
- Initial dirty state: untracked `specs/405-json-to-jsonb-data-layer-hardening/`
- Session branch not created because the active spec package was already untracked in the working tree; work continued cautiously on the current feature branch.
- Initial `git diff --check`: passed.
Final `git status --short --untracked-files=all`:
```text
M apps/platform/app/Models/BackupItem.php
?? apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php
?? apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php
?? specs/405-json-to-jsonb-data-layer-hardening/checklists/requirements.md
?? specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md
?? specs/405-json-to-jsonb-data-layer-hardening/plan.md
?? specs/405-json-to-jsonb-data-layer-hardening/spec.md
?? specs/405-json-to-jsonb-data-layer-hardening/tasks.md
```
The spec package was untracked at session start. This implementation added the migration, database test, implementation report, task completion updates, and the `BackupItem` runtime adjustment.
## 4. JSON/JSONB Inventory Matrix
Inventory source: PostgreSQL `information_schema.columns`, row/null counts from the live local database, 2026-06-23. Converted-column migration decision is direct `ALTER COLUMN ... TYPE jsonb USING ...::jsonb`, grouped per table when more than one column is converted on the same table. Local converted tables have small row counts and no JSON-column index rebuild requirement; staging must still validate actual lock duration and migration runtime before production promotion.
| Column | Type | Rows | Nulls | Nullable | Default | Indexes | Classification | Decision |
|---|---:|---:|---:|---|---|---|---|---|
| alert_deliveries.payload | json | 0 | 0 | YES | none | none | CONVERT P2 | Alert delivery payload; direct conversion. |
| alert_rules.tenant_allowlist | json | 0 | 0 | YES | none | none | CONVERT P2 | Alert scoping list; direct conversion. |
| audit_logs.metadata | json | 1320 | 0 | YES | none | none | CONVERT P1 | Audit context queried by metadata key; direct conversion. |
| backup_items.payload | json | 246 | 0 | NO | none | none | CONVERT P1 | Critical backup snapshot payload; direct conversion. |
| backup_items.metadata | json | 246 | 0 | YES | none | none | CONVERT P1 | Backup metadata and warnings; direct conversion. |
| backup_items.assignments | json | 246 | 146 | YES | none | none | CONVERT P1 | Assignment proof payload; direct conversion and query function update. |
| backup_schedules.days_of_week | json | 0 | 0 | YES | none | none | CONVERT P2 | Schedule structured config; direct conversion. |
| backup_schedules.policy_types | json | 0 | 0 | NO | none | none | CONVERT P2 | Schedule policy-type filter; direct conversion. |
| backup_sets.metadata | json | 25 | 0 | YES | none | none | CONVERT P1 | Backup-set provenance metadata; direct conversion. |
| baseline_profiles.scope_jsonb | jsonb | 4 | 0 | NO | none | none | ALREADY_JSONB | No migration. |
| baseline_snapshot_items.meta_jsonb | jsonb | 102 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| baseline_snapshots.summary_jsonb | jsonb | 4 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| baseline_snapshots.completion_meta_jsonb | jsonb | 4 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| baseline_tenant_assignments.override_scope_jsonb | jsonb | 3 | 3 | YES | none | none | ALREADY_JSONB | No migration. |
| entra_groups.group_types | jsonb | 574 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| environment_review_sections.summary_payload | jsonb | 153 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| environment_review_sections.render_payload | jsonb | 153 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| environment_reviews.summary | jsonb | 28 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| evidence_snapshot_items.summary_payload | jsonb | 98 | 0 | NO | `{}` | `evidence_snapshot_items_payload_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| evidence_snapshots.summary | jsonb | 24 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| finding_exception_decisions.metadata | jsonb | 7 | 0 | NO | `{}` | `finding_exception_decisions_metadata_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| finding_exception_evidence_references.summary_payload | jsonb | 0 | 0 | NO | `{}` | `finding_exception_evidence_refs_payload_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| finding_exceptions.evidence_summary | jsonb | 9 | 0 | NO | `{}` | `finding_exceptions_evidence_summary_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| findings.evidence_jsonb | jsonb | 254 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| inventory_items.meta_jsonb | jsonb | 229 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| inventory_links.metadata | jsonb | 251 | 0 | YES | none | none | ALREADY_JSONB | No migration. |
| managed_environment_onboarding_sessions.state | json | 2 | 0 | YES | none | none | CONVERT P1 | Onboarding state payload; direct conversion. |
| managed_environment_permissions.details | json | 135 | 0 | YES | none | none | CONVERT P1 | Provider permission details; direct conversion. |
| managed_environment_triage_reviews.review_snapshot | jsonb | 0 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| managed_environments.metadata | json | 54 | 2 | YES | none | none | CONVERT P1 | Provider/environment metadata; direct conversion. |
| managed_environments.rbac_canary_results | json | 54 | 54 | YES | none | none | CONVERT P1 | RBAC readiness proof; direct conversion. |
| managed_environments.rbac_last_warnings | json | 54 | 54 | YES | none | none | CONVERT P1 | RBAC warning proof; direct conversion. |
| notifications.data | jsonb | 311 | 0 | NO | none | none | ALREADY_JSONB | No migration. |
| operation_runs.summary_counts | jsonb | 96 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| operation_runs.failure_summary | jsonb | 96 | 0 | NO | `[]` | none | ALREADY_JSONB | No migration. |
| operation_runs.context | jsonb | 96 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| platform_users.capabilities | jsonb | 1 | 0 | NO | `[]` | none | ALREADY_JSONB | No migration. |
| policies.metadata | json | 208 | 3 | YES | none | none | CONVERT P1 | Policy inventory metadata; direct conversion. |
| policy_versions.snapshot | json | 422 | 0 | NO | none | none | CONVERT P1 | Immutable policy snapshot; direct conversion. |
| policy_versions.metadata | json | 422 | 0 | YES | none | none | CONVERT P1 | Version metadata; direct conversion. |
| policy_versions.assignments | json | 422 | 216 | YES | none | none | CONVERT P1 | Version assignment snapshot; direct conversion. |
| policy_versions.scope_tags | json | 422 | 14 | YES | none | none | CONVERT P1 | Scope tag snapshot; direct conversion. |
| policy_versions.secret_fingerprints | json | 422 | 2 | YES | none | none | CONVERT P1 | Redaction integrity metadata; direct conversion. |
| product_usage_events.metadata | jsonb | 100 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| provider_connections.scopes_granted | jsonb | 16 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| provider_connections.metadata | jsonb | 16 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| restore_runs.requested_items | json | 4 | 2 | YES | none | none | CONVERT P1 | Restore request payload; direct conversion. |
| restore_runs.preview | json | 4 | 1 | YES | none | none | CONVERT P1 | Restore preview payload; direct conversion. |
| restore_runs.results | json | 4 | 0 | YES | none | none | CONVERT P1 | Restore execution result payload; direct conversion. |
| restore_runs.metadata | json | 4 | 0 | YES | none | none | CONVERT P1 | Restore metadata; direct conversion. |
| restore_runs.group_mapping | json | 4 | 4 | YES | none | none | CONVERT P1 | Restore group mapping; direct conversion. |
| review_packs.summary | jsonb | 23 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| review_packs.options | jsonb | 23 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| review_publication_resolution_cases.summary | jsonb | 3 | 0 | NO | `{}` | `review_publication_resolution_cases_summary_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| review_publication_resolution_cases.metadata | jsonb | 3 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| review_publication_resolution_steps.summary | jsonb | 17 | 0 | NO | `{}` | `review_publication_resolution_steps_summary_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| review_publication_resolution_steps.metadata | jsonb | 17 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| settings_catalog_definitions.raw | jsonb | 0 | 0 | NO | none | `idx_settings_catalog_definitions_raw_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| stored_reports.payload | jsonb | 35 | 0 | NO | none | `stored_reports_payload_gin` | ALREADY_JSONB | Existing justified GIN retained. |
| support_requests.context_envelope | jsonb | 0 | 0 | NO | `{}` | none | ALREADY_JSONB | No migration. |
| tenant_settings.value | json | 0 | 0 | NO | none | none | CONVERT P1 | Tenant-scoped setting value; direct conversion. |
| workspace_settings.value | json | 0 | 0 | NO | none | none | CONVERT P1 | Workspace-scoped setting value; direct conversion. |
No live `json` column was left as `KEEP_JSON`, `DEPRECATED`, or `DECISION_REQUIRED`: every live `json` column had current model ownership, bounded row counts, existing cast/read semantics, and a trust-layer storage reason to align with the existing `jsonb` baseline.
Inventory ownership, cast, and query continuation for converted columns:
| Column | Model / owner | Existing cast or accessor | Existing query/rendered usage | Constraint/FK context |
|---|---|---|---|---|
| alert_deliveries.payload | `App\Models\AlertDelivery` | `array` | Alert delivery rendering/notification payload; no JSON predicate. | Workspace/environment/rule/destination FKs unchanged. |
| alert_rules.tenant_allowlist | `App\Models\AlertRule` | `array` | Alert tenant scoping list; no JSON predicate. | Workspace FK unchanged. |
| audit_logs.metadata | `App\Models\AuditLog` / `AuditRecorder` | `array` | `metadata ->> '_dedupe_key'`, `metadata->...` audit filters/tests. | Workspace/environment/operation FKs and audit scope check unchanged. |
| backup_items.payload | `App\Models\BackupItem` | `array` | Backup set detail, restore readiness, included item rendering. | Workspace/environment/backup/policy FKs unchanged. |
| backup_items.metadata | `App\Models\BackupItem` | `array` | Backup quality/source/warning helpers and rendered backup detail. | Workspace/environment/backup/policy FKs unchanged. |
| backup_items.assignments | `App\Models\BackupItem` | `array` | `scopeWithAssignments()`; assignment count/group helper rendering. | Workspace/environment/backup/policy FKs unchanged. |
| backup_schedules.days_of_week | `App\Models\BackupSchedule` | `array` | Schedule form/detail semantics; no JSON predicate. | Workspace/environment FKs and frequency check unchanged. |
| backup_schedules.policy_types | `App\Models\BackupSchedule` | `array` | Schedule policy-type selection; no JSON predicate. | Workspace/environment FKs and frequency check unchanged. |
| backup_sets.metadata | `App\Models\BackupSet` | `array` | Backup provenance/source helpers; `metadata->source` test path. | Workspace/environment FK unchanged. |
| managed_environment_onboarding_sessions.state | `App\Models\ManagedEnvironmentOnboardingSession` | `array` with allowed-key mutator | Onboarding resume/state rendering and historical migration reads. | Workspace/environment/user FKs and lifecycle constraints unchanged. |
| managed_environment_permissions.details | `App\Models\ManagedEnvironmentPermission` | `array` | Provider permission/readiness detail rendering; no JSON predicate. | Workspace/environment FK and permission unique key unchanged. |
| managed_environments.metadata | `App\Models\ManagedEnvironment` | `array` | Environment/provider metadata accessors and rendered environment context. | Workspace FK and tenant lifecycle constraints unchanged. |
| managed_environments.rbac_canary_results | `App\Models\ManagedEnvironment` | JSON decode accessor | RBAC readiness proof rendering. | Workspace FK and tenant lifecycle constraints unchanged. |
| managed_environments.rbac_last_warnings | `App\Models\ManagedEnvironment` | JSON decode accessor | RBAC warning rendering. | Workspace FK and tenant lifecycle constraints unchanged. |
| policies.metadata | `App\Models\Policy` | `array` | Policy inventory metadata rendering and helper access. | Workspace/environment FK unchanged. |
| policy_versions.snapshot | `App\Models\PolicyVersion` | `array` | Immutable version snapshot rendering/diff/backup provenance. | Workspace/environment/policy FKs unchanged. |
| policy_versions.metadata | `App\Models\PolicyVersion` | `array` | Version source/warning/integrity helpers. | Workspace/environment/policy FKs unchanged. |
| policy_versions.assignments | `App\Models\PolicyVersion` | `array` | Version assignment snapshot rendering; sibling hash index unchanged. | Workspace/environment/policy FKs unchanged. |
| policy_versions.scope_tags | `App\Models\PolicyVersion` | `array` | Scope tag snapshot rendering; sibling hash index unchanged. | Workspace/environment/policy FKs unchanged. |
| policy_versions.secret_fingerprints | `App\Models\PolicyVersion` | `array` | Redaction integrity helper. | Workspace/environment/policy FKs unchanged. |
| restore_runs.requested_items | `App\Models\RestoreRun` | `array` | Restore request/preview detail rendering. | Workspace/environment/backup FKs unchanged. |
| restore_runs.preview | `App\Models\RestoreRun` | `array` | Restore preview wizard/detail rendering. | Workspace/environment/backup FKs unchanged. |
| restore_runs.results | `App\Models\RestoreRun` | `array` | Restore result proof/detail rendering. | Workspace/environment/backup FKs unchanged. |
| restore_runs.metadata | `App\Models\RestoreRun` | `array` | Restore result/safety metadata and reconciliation helpers; no direct JSON predicate on this table. | Workspace/environment/backup FKs unchanged. |
| restore_runs.group_mapping | `App\Models\RestoreRun` | `array` | Restore group-mapping preview/detail rendering. | Workspace/environment/backup FKs unchanged. |
| tenant_settings.value | `App\Models\TenantSetting` | `array` | Tenant settings read/write; no JSON predicate. | Workspace/environment/user FKs unchanged. |
| workspace_settings.value | `App\Models\WorkspaceSetting` | `array` | Workspace settings read/write; no JSON predicate. | Workspace/user FKs unchanged. |
Already-JSONB ownership groups were also reviewed and left unchanged:
- Baseline/inventory/finding/evidence/review rows: existing `*_jsonb`, `summary`, `summary_payload`, `render_payload`, `meta_jsonb`, and evidence payload columns with current model/service ownership; existing GIN indexes retained where already present.
- OperationRun/provider/notification/support rows: existing `operation_runs.context`, summary/failure payloads, provider connection metadata/scopes, notification data, product usage metadata, and support request context already use `jsonb`.
- Stored report and review publication rows: `stored_reports.payload`, review pack summaries/options, and review publication resolution summaries/metadata already use `jsonb`; browser proof confirms rendered output remains stable.
## 5. Migrations Added
Added `apps/platform/database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php`.
- Converts the 27 reviewed `CONVERT` columns with `ALTER TABLE ... ALTER COLUMN ... TYPE jsonb USING column::jsonb`.
- Groups multiple converted columns on the same table into a single `ALTER TABLE` statement to avoid repeated per-column table rewrites/lock windows where PostgreSQL can satisfy the changes together.
- Runs only when `DB::getDriverName() === 'pgsql'`.
- Preserves nullability, defaults, constraints, and existing non-JSON indexes by altering only the column type.
- Rollback converts the same columns back to `json` with `USING column::json`.
- Rollback limitation: `jsonb` canonicalizes object key ordering and duplicate JSON object keys. Semantic content is preserved, but byte-for-byte textual JSON representation is not guaranteed.
- Local rollback/down-up proof passed in the PostgreSQL lane, including `BackupItem::withAssignments()` while the column is temporarily `json`.
## 6. Index Justification
No new JSONB indexes were added.
Usage scan found one conversion-affected raw JSON function path, `BackupItem::scopeWithAssignments()`, which filters by array length but does not justify a GIN or expression index at current row counts. Existing hash/sibling indexes and existing GIN indexes on already-JSONB tables remain unchanged.
The new PostgreSQL test asserts no speculative GIN index exists on converted-column tables.
## 7. Runtime Changes Made
Changed `apps/platform/app/Models/BackupItem.php`:
- `scopeWithAssignments()` now uses `jsonb_array_length(assignments::jsonb)` on PostgreSQL.
- The explicit cast keeps the scope compatible during pre-migration deploy windows and after a database-only rollback to `json`.
- Non-PostgreSQL test lanes keep `json_array_length(assignments)` to avoid breaking SQLite-style JSON test support.
No model casts required changes. Laravel array casts continue to read/write `jsonb` columns correctly.
## 8. Tests Added or Updated
Added `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php` with PostgreSQL-only coverage:
- All reviewed legacy `json` columns are `jsonb` after migrations.
- The 405 migration can run `down()` to `json` and back `up()` to `jsonb` while preserving representative existing rows.
- Nullability, defaults, table indexes, and constraints remain unchanged across the 405 down/up cycle.
- No remaining live public-schema `json` columns remain after the conversion migration.
- No speculative GIN index was introduced for converted columns.
- Representative read/write/cast behavior for alert, audit, backup, restore, policy, provider/environment, settings, and onboarding payload categories.
- Audit metadata key query using `metadata ->> '_dedupe_key'`.
- `BackupItem::withAssignments()` query path against both rollback-state `json` and migrated `jsonb`.
Latest result: 5 tests passed, 260 assertions.
Existing test run:
- `apps/platform/tests/Unit/BackupItemTest.php` still passes after the driver-aware scope change.
- Latest result: 20 tests passed, 31 assertions.
## 9. Data Validation
Local PostgreSQL validation:
- The migration/type test ran migrations on the PostgreSQL testing database and asserted every converted column is `jsonb`.
- The new down/up test temporarily rolled the 405 migration back to `json`, inserted representative rows across every converted column category, verified `BackupItem::withAssignments()` works in the rollback-state schema, migrated forward to `jsonb`, and compared decoded payload semantics.
- The same down/up test asserted nullability, defaults, table index definitions, and table constraint definitions remain unchanged after conversion.
- The same test asserted there are no remaining public-schema columns with `data_type = 'json'`.
- Representative non-sensitive samples were written and read back through Eloquent casts and direct JSONB key predicates.
- Row/null counts were recorded before conversion from the local development database; converted tables are small enough for direct `ALTER COLUMN` locally. Staging/Dokploy must still validate real runtime and lock behavior.
Not performed in this session:
- Production-sized timing.
- Staging/Dokploy migration runtime.
- Full byte-level dump comparison. This is not required because `jsonb` canonicalizes textual representation; semantic preservation was tested instead.
## 10. Browser Proof
Focused browser proof passed using existing payload-backed smoke tests. No UI files were changed; this proof is regression-only.
Command:
```bash
cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Browser/Spec403EvidenceCurrentnessRuntimeClosureSmokeTest.php tests/Browser/Spec394ProviderFreshnessPermissionSmokeTest.php tests/Browser/Spec371BackupSetProductizationSmokeTest.php tests/Browser/Spec335RestoreRunDetailProductizationSmokeTest.php tests/Browser/Spec379ManagementReportPdfSmokeTest.php
```
Result:
```text
5 browser tests passed, 176 assertions, 19.87s
```
Covered surfaces:
- Evidence Overview and OperationRun proof currentness: `Spec403EvidenceCurrentnessRuntimeClosureSmokeTest.php`
- Provider freshness and required-permission readiness: `Spec394ProviderFreshnessPermissionSmokeTest.php`
- Backup set list/detail decision hierarchy and included backup items: `Spec371BackupSetProductizationSmokeTest.php`
- Restore run detail post-execution proof/results rendering: `Spec335RestoreRunDetailProductizationSmokeTest.php`
- Review Pack management PDF / Stored Report output state: `Spec379ManagementReportPdfSmokeTest.php`
Browser tests asserted no JavaScript errors and no unexpected console logs on the tested surfaces.
## 11. Regression Proof
Completed:
- PostgreSQL schema/type/index/model/query proof passed.
- PostgreSQL rollback/down-up preservation proof passed.
- Existing `BackupItem` unit regression suite passed.
- Focused browser proof passed across evidence, operations, provider, backup, restore, review pack, stored report, and management PDF surfaces.
- No UI/runtime surface files were edited.
- No authorization, global search, provider registration, destructive/high-impact action, or asset behavior was changed.
## 12. Staging Validation
Staging/Dokploy validation is not accessible from this agent session.
Required staging release checks before production:
- Run the migration on staging PostgreSQL.
- Confirm app boot.
- Run the focused PostgreSQL Spec405 test or equivalent staging validation.
- Smoke the representative payload-backed surfaces.
- Confirm no secrets or raw provider credential payloads appear in logs/screenshots.
Because staging is unavailable here, final readiness remains `PASS WITH CONDITIONS`.
## 13. Remaining Findings
- No P0 findings.
- No unresolved `json` column classification.
- No in-scope schema or application regression currently known after fixing rollback-compatible query handling and grouped table conversion.
- Remaining P1 condition: staging/Dokploy validation unavailable in this session.
- Browser proof passed locally.
## 14. Deferred Items
- Staging/Dokploy validation and migration runtime observation.
- Optional future online migration strategy if a production-sized table proves too large for direct `ALTER COLUMN`.
- Full browser/runtime audit remains out of scope.
- Governance artifact lifecycle and retention remain separate follow-up scope.
## 15. Validation Commands
Passed:
```bash
git diff --check
cd apps/platform && ./vendor/bin/sail php vendor/bin/pest -c phpunit.pgsql.xml tests/Feature/Database/JsonbDataLayerHardeningTest.php
cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Unit/BackupItemTest.php
cd apps/platform && ./vendor/bin/sail pint app/Models/BackupItem.php database/migrations/2026_06_23_000405_convert_trust_payload_json_columns_to_jsonb.php tests/Feature/Database/JsonbDataLayerHardeningTest.php
cd apps/platform && ./vendor/bin/sail php vendor/bin/pest tests/Browser/Spec403EvidenceCurrentnessRuntimeClosureSmokeTest.php tests/Browser/Spec394ProviderFreshnessPermissionSmokeTest.php tests/Browser/Spec371BackupSetProductizationSmokeTest.php tests/Browser/Spec335RestoreRunDetailProductizationSmokeTest.php tests/Browser/Spec379ManagementReportPdfSmokeTest.php
```
## 16. Product Surface, Filament, Deployment, and Recommended Next Step Close-Out
- **Application implementation status**: implemented locally with focused PostgreSQL rollback/down-up, unit, formatter, diff, and browser proof passing; staging validation still conditions the gate.
- **Livewire v4 compliance**: Livewire 4.1.4 confirmed by Laravel Boost; no Livewire code changed.
- **Provider registration location**: Laravel 12 panel providers remain in `apps/platform/bootstrap/providers.php`; no provider registration changed.
- **Global search posture**: unchanged. No globally searchable resource was added or modified.
- **Destructive/high-impact actions**: none added or changed; no confirmation/authorization behavior changed.
- **Asset strategy**: no frontend assets added, no `FilamentAsset` registration, no new `filament:assets` deploy requirement beyond the existing deployment baseline.
- **Product Surface Impact**: no runtime UI surface changed.
- **UI Surface Impact**: none; focused browser proof is regression-only.
- **No-legacy posture**: canonical data-layer conversion; no compatibility shim or dual-write path introduced.
- **Page archetype / surface budgets / Technical Annex / deep-link demotion / canonical status vocabulary**: N/A for changed code; existing surfaces remain unchanged.
- **Product Surface exceptions**: none.
- **Focused browser proof**: passed across evidence/currentness, operations, provider readiness, backup, restore, review pack, stored report, and management PDF surfaces.
- **Human Product Sanity**: visible complexity unchanged; no raw payload exposure added; current/released/failed/partial semantics unchanged.
- **Implementation-report fields**: Livewire v4, provider registration, global search, destructive/high-impact actions, asset strategy, deployment impact, tests/browser result, and visible complexity are recorded here.
- **Deployment impact**: database migration only. No env vars, queues, scheduler, storage, routes, provider scopes, panels, assets, or worker changes. Staging validation is required before production promotion.
- **No completed-spec rewrite assertion**: Specs 400-404 were read-only context; their files were not rewritten.
- **Recommended next step**: run staging PostgreSQL migration validation before production promotion.

401
specs/405-json-to-jsonb-data-layer-hardening/plan.md Normal file
View File

@ -0,0 +1,401 @@
# Implementation Plan: Spec 405 - JSON-to-JSONB Data-layer Hardening
**Branch**: `405-json-to-jsonb-data-layer-hardening` | **Date**: 2026-06-23 | **Spec**: `specs/405-json-to-jsonb-data-layer-hardening/spec.md`
**Input**: Feature specification from `specs/405-json-to-jsonb-data-layer-hardening/spec.md`
## Summary
Prepare and later implement a database-hardening slice that inventories every live PostgreSQL `json` and `jsonb` column, classifies all `json` columns, converts only appropriate queryable trust-layer payload columns to `jsonb`, adds only query-backed indexes, proves semantic payload preservation, and records local plus staging-like validation. The implementation must not add product behavior, UI surfaces, broad abstractions, normalized replacement tables, or speculative indexes.
## Technical Context
**Language/Version**: PHP 8.4.15, Laravel 12.52.0, Filament 5.2.1, Livewire 4.1.4.
**Primary Dependencies**: Laravel migrations/schema builder, PostgreSQL, Pest 4, Filament/Livewire test helpers where focused browser or rendered-surface proof is needed.
**Storage**: PostgreSQL via Sail/Dokploy; target change is existing column type conversion from `json` to `jsonb` where classified `CONVERT`.
**Testing**: Pest 4, PostgreSQL lane, targeted feature tests, focused browser smoke.
**Validation Lanes**: pgsql, confidence/feature, focused browser, optional profiling/explain for new indexes.
**Target Platform**: TenantPilot Laravel monolith under `apps/platform`; Spec Kit artifacts under `specs/405-json-to-jsonb-data-layer-hardening/`.
**Project Type**: Laravel web application plus Spec Kit workflow.
**Performance Goals**: Avoid speculative indexes; each new index must have an existing query path and bounded write-overhead rationale.
**Constraints**: No new product concepts, UI surfaces, Graph calls, authorization model, lifecycle behavior, normalized replacement tables, provider semantics, broad abstractions, or completed-spec rewrites.
**Scale/Scope**: All live PostgreSQL `json` and `jsonb` columns; conversion limited to selected existing columns with proof.
## Repository Truth And Initial Signals
Laravel Boost confirmed PostgreSQL and current packages:
```text
PHP 8.4.15
Laravel 12.52.0
Filament 5.2.1
Livewire 4.1.4
Pest 4.3.1
PostgreSQL
```
Laravel 12 migrations support `$table->jsonb('column')`; implementation should use Laravel migrations where possible and raw PostgreSQL `ALTER TABLE ... ALTER COLUMN ... TYPE jsonb USING ...::jsonb` where column type alteration requires explicit SQL.
Live PostgreSQL schema inspection on 2026-06-23 found active `json` columns requiring classification in:
```text
alert_deliveries.payload
alert_rules.tenant_allowlist
audit_logs.metadata
backup_items.assignments
backup_items.metadata
backup_items.payload
backup_schedules.days_of_week
backup_schedules.policy_types
backup_sets.metadata
managed_environment_onboarding_sessions.state
managed_environment_permissions.details
managed_environments.metadata
managed_environments.rbac_canary_results
managed_environments.rbac_last_warnings
policies.metadata
policy_versions.assignments
policy_versions.metadata
policy_versions.scope_tags
policy_versions.secret_fingerprints
policy_versions.snapshot
restore_runs.group_mapping
restore_runs.metadata
restore_runs.preview
restore_runs.requested_items
restore_runs.results
tenant_settings.value
workspace_settings.value
```
Newer trust-layer paths already use `jsonb`, including `operation_runs.summary_counts`, `operation_runs.failure_summary`, `operation_runs.context`, baseline `scope_jsonb`/`summary_jsonb`/`meta_jsonb`, evidence summaries, findings evidence, review pack summaries/options, stored report payloads, provider connection metadata/scopes, and review publication resolution payloads.
## Technical Approach
1. Record branch, HEAD, dirty state, and `git diff --check`.
2. Query PostgreSQL `information_schema.columns` for all `json` and `jsonb` columns and collect row counts, null counts, defaults, indexes, and constraints.
3. Map each column to model casts, factories/fixtures, query usages, Filament/rendered usage, tests, and sensitive-data boundaries.
4. Classify each column as `CONVERT`, `KEEP_JSON`, `ALREADY_JSONB`, `DEPRECATED`, or `DECISION_REQUIRED`.
5. Write one or more focused migrations converting only `CONVERT` columns.
6. Add only query-backed JSONB indexes with explicit proof and rollback/drop strategy.
7. Update casts or query code only if required by tests after conversion.
8. Add PostgreSQL/feature tests proving type conversion, semantic preservation, model read/write behavior, scope/authorization boundaries, and representative domain regressions.
9. Run focused browser proof over existing payload-backed surfaces.
10. Produce `implementation-report.md` with the inventory matrix, validation results, remaining findings, staging status, and final gate result.
## Likely Affected Repository Surfaces
Preparation identifies likely inspection and implementation surfaces; implementation must verify exact paths before editing.
```text
apps/platform/database/migrations/
apps/platform/app/Models/
apps/platform/database/factories/
apps/platform/tests/
apps/platform/app/Support/
apps/platform/app/Services/
apps/platform/app/Jobs/
apps/platform/app/Filament/
apps/platform/resources/views/
specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md
```
Likely model/cast inspection targets include:
```text
App\Models\AlertDelivery
App\Models\AlertRule
App\Models\AuditLog
App\Models\BackupItem
App\Models\BackupSchedule
App\Models\BackupSet
App\Models\ManagedEnvironment
App\Models\ManagedEnvironmentOnboardingSession
App\Models\ManagedEnvironmentPermission
App\Models\Policy
App\Models\PolicyVersion
App\Models\RestoreRun
App\Models\TenantSetting
App\Models\WorkspaceSetting
```
## UI / Surface Guardrail Plan
- **Guardrail scope**: backend data-layer hardening with focused rendered regression proof.
- **Affected routes/pages/actions/states/navigation/panel/provider surfaces**: none changed by implementation unless the spec is updated first.
- **No-impact class**: backend-only schema/storage conversion.
- **Native vs custom classification summary**: N/A.
- **Shared-family relevance**: evidence/report viewers, provider readiness, OperationRun proof, backup/restore proof, and audit metadata are regression consumers only.
- **State layers in scope**: persistence and existing rendered state regression only.
- **Audience modes in scope**: operator/MSP and customer/read-only only for regression proof; no disclosure behavior changes.
- **Decision/diagnostic/raw hierarchy plan**: unchanged; raw payloads remain technical/audit detail.
- **Raw/support gating plan**: unchanged.
- **One-primary-action / duplicate-truth control**: unchanged.
- **Handling modes by drift class or surface**: report-only unless conversion causes a confirmed regression, then minimal in-scope fix.
- **Repository-signal treatment**: review-mandatory for changed database/runtime paths.
- **Special surface test profiles**: backend storage conversion plus focused browser proof.
- **Required tests or manual smoke**: PostgreSQL migration tests, feature regressions, focused browser smoke.
- **Exception path and spread control**: any required UI edit is out of scope and must stop implementation for spec/plan update.
- **Active feature PR close-out entry**: Guardrail / database hardening.
- **UI/Productization coverage decision**: No UI surface impact.
- **Coverage artifacts to update**: none unless implementation unexpectedly changes rendered UI; then stop and update spec/plan first.
- **No-impact rationale**: storage type conversion and proof only.
- **Navigation / Filament provider-panel handling**: no panel/provider changes.
- **Screenshot or page-report need**: focused browser proof may produce screenshots/logs as evidence; no page report required without UI changes.
## Product Surface Contract Plan
- **Product Surface Contract reference**: `docs/product/standards/product-surface-contract.md` as regression lens only.
- **No-legacy posture**: canonical replacement; no compatibility shims or dual-write paths.
- **Page archetype and surface budget plan**: N/A for changed code; browser proof names existing inspected archetypes.
- **Technical Annex and deep-link demotion plan**: unchanged; conversion must not expose raw payloads, internal IDs, OperationRun links, or evidence deep links by default.
- **Canonical status vocabulary plan**: unchanged.
- **Product Surface exceptions**: none.
- **Browser verification plan**: focused existing-surface regression proof required.
- **Human Product Sanity plan**: final implementation report confirms unchanged trust semantics.
- **Visible complexity outcome target**: neutral.
- **Implementation report target**: `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`.
## Filament / Livewire / Deployment Posture
- **Livewire v4 compliance**: Livewire 4.1.4 confirmed; no Livewire code change planned.
- **Panel provider registration location**: Laravel 12 panel providers remain in `apps/platform/bootstrap/providers.php`; no panel change.
- **Global search posture**: no resource global search posture changed. If implementation touches a resource unexpectedly, verify View/Edit/global search safety or stop for spec update.
- **Destructive/high-impact action posture**: no actions added or changed. Existing destructive/high-impact action proof from Specs 401-404 must not regress.
- **Asset strategy**: no assets, no `FilamentAsset` registration, no new `filament:assets` requirement beyond existing deployment baseline.
- **Testing plan**: database/migration tests, feature/domain regressions, focused browser proof for existing payload-backed surfaces.
- **Deployment impact**: migrations and staging/Dokploy validation. No env vars, queues, scheduler, storage, assets, routes, provider scopes, or panel providers planned.
## Shared Pattern & System Fit
- **Cross-cutting feature marker**: yes at data storage level.
- **Systems touched**: database schema, existing casts, existing query paths, tests, implementation report.
- **Shared abstractions reused**: Laravel migrations, existing models/casts, existing scoped query paths, existing tests/browser fixtures.
- **New abstraction introduced? why?**: none planned.
- **Why the existing abstraction was sufficient or insufficient**: existing models and services already own payload meaning; storage conversion does not require a new runtime layer.
- **Bounded deviation / spread control**: any new helper must be justified as a narrow test/support helper, not a runtime framework.
## OperationRun UX Impact
- **Touches OperationRun start/completion/link UX?**: no.
- **Central contract reused**: N/A.
- **Delegated UX behaviors**: N/A.
- **Surface-owned behavior kept local**: N/A.
- **Queued DB-notification policy**: N/A.
- **Terminal notification path**: N/A.
- **Exception path**: none.
## Provider Boundary & Portability Fit
- **Shared provider/platform boundary touched?**: storage inspection only.
- **Provider-owned seams**: provider raw/permission/readiness payload keys remain unchanged.
- **Platform-core seams**: storage type, workspace/managed-environment scope, audit/report/evidence ownership, migration safety.
- **Neutral platform terms / contracts preserved**: workspace, managed environment, provider, connection, operation, evidence, report, backup, restore.
- **Retained provider-specific semantics and why**: existing Microsoft/Intune payload keys remain provider-owned payload content.
- **Bounded extraction or follow-up path**: none for this spec.
## Domain / Model Implications
- No new model, table, persisted artifact, enum, status family, route, action, provider type, or source of truth is introduced.
- Existing model casts may remain unchanged because Laravel treats `json` and `jsonb` similarly at the application layer; implementation must update casts only if tests prove current behavior breaks.
- Existing raw payload keys and product meaning remain unchanged.
- Columns with unclear ownership or product semantics must be classified `DECISION_REQUIRED`, not converted by assumption.
## Data / Migration Implications
Migration pattern for direct conversions:
```sql
ALTER TABLE table_name
ALTER COLUMN column_name TYPE jsonb
USING column_name::jsonb;
```
Rollback pattern where feasible:
```sql
ALTER TABLE table_name
ALTER COLUMN column_name TYPE json
USING column_name::json;
```
Implementation must preserve:
- nullable state
- defaults
- constraints
- existing indexes unless intentionally replaced
- row counts
- application read/write behavior
Implementation must document rollback limitations:
- `jsonb` normalizes key order
- duplicate JSON object keys may be normalized
- semantic content must remain preserved, but textual representation may differ
## Index Strategy
Allowed only with existing query proof:
- GIN index for existing containment/query usage.
- Expression index for existing frequently queried JSONB key.
- Partial index for existing filtered JSONB key access.
Rejected:
- index every converted column
- index because `jsonb` supports it
- index for future lifecycle/export/dashboard guesses
- index without write-overhead note
Each index requires a row in the implementation report:
```text
Index | Table/Column | Query Path | Reason | Expected Benefit | Write Overhead Risk | Proof
```
## RBAC / Security / Audit Implications
- No RBAC behavior changes.
- Any changed query involving payload keys must retain existing workspace and managed-environment scoping.
- Non-member and wrong-scope access must remain deny-as-not-found where applicable.
- Customer-safe report/review/evidence paths must not expose raw payloads because of conversion or debug output.
- Reports, logs, screenshots, and test fixtures must not include secrets, tokens, raw credential payloads, sensitive raw provider payloads, or customer-sensitive raw payloads.
- Audit metadata conversion must preserve actor/context fields.
## OperationRun / Evidence / Result Truth Implications
The plan distinguishes:
- **Execution truth**: existing `OperationRun` status/outcome/summary/context; live columns are already `jsonb`, but regression proof still checks rendering.
- **Artifact truth**: `ReviewPack`, `StoredReport`, evidence snapshots/items, backup sets/items.
- **Backup/snapshot truth**: policy versions, backup payloads, baseline/evidence payloads.
- **Recovery/evidence truth**: restore previews/results, evidence currentness, report receipts.
- **Operator next action**: unchanged existing UI states and actions.
## Test Strategy
Required test groups:
1. PostgreSQL schema/type tests for every converted column.
2. Migration semantic preservation tests for representative non-sensitive payloads.
3. Model cast/read-write tests where converted columns are written by Eloquent models.
4. Query-path tests for any changed JSON key query and any new index.
5. Evidence/currentness regression tests for converted evidence/report/review payloads.
6. OperationRun/audit regression tests where summary/context/audit metadata is in scope.
7. Provider readiness/freshness/permission regression tests where provider/environment payloads are converted.
8. Backup/restore payload regression tests for backup items/sets/schedules and restore preview/results.
9. Review/report receipt regression tests for review pack/stored report/customer output.
10. Authorization/scope tests for changed payload queries.
11. Focused browser smoke for representative existing payload-backed pages.
Preferred validation commands:
```bash
cd apps/platform && ./vendor/bin/sail php vendor/bin/pest -c phpunit.pgsql.xml --filter=Spec405
cd apps/platform && ./vendor/bin/sail artisan test --filter=Evidence
cd apps/platform && ./vendor/bin/sail artisan test --filter=OperationRun
cd apps/platform && ./vendor/bin/sail artisan test --filter=Provider
cd apps/platform && ./vendor/bin/sail artisan test --filter=Backup
cd apps/platform && ./vendor/bin/sail artisan test --filter=Restore
cd apps/platform && ./vendor/bin/sail artisan test --filter=ReviewPack
cd apps/platform && ./vendor/bin/sail artisan test --filter=StoredReport
```
Use narrower commands where implementation creates focused Spec 405 tests.
## Rollout And Deployment Considerations
- Local: run through Laravel Sail against PostgreSQL.
- Staging: validate migration execution, app boot, queue/browser relevant proof, representative pages, and rollback/forward notes where safe.
- Production: do not claim production readiness unless staging-like validation passes or the final report explicitly records `PASS WITH CONDITIONS`.
- Migrations: assess table size/row count and lock risk before direct conversion.
- Env vars: none expected.
- Queues/scheduler/storage/assets: none expected, but affected flows may rely on existing workers/storage and must not regress.
- Dokploy: database migration and app boot validation required before production promotion.
## Risk Controls
- Stop before migration if inventory is incomplete.
- Stop before conversion if a high-risk column has unclear product ownership.
- Use semantic JSON comparisons, not raw string comparisons.
- Classify large-table conversion risk before direct `ALTER COLUMN`.
- Do not add speculative indexes.
- Do not print sensitive payload samples.
- Do not rewrite completed specs.
- If a conversion causes UI/rendered regression, fix the data-layer cause if bounded; otherwise stop for spec update.
## Constitution Check
- Inventory-first: PASS; inventory and source-of-truth classification precede conversion.
- Read/write separation: PASS; no Graph/write product behavior is added.
- Graph contract path: PASS; no Graph calls are added.
- Deterministic capabilities: N/A; no capability resolver changes.
- RBAC-UX: PASS; authorization must not change and changed payload queries require scope tests.
- Workspace isolation: PASS; scoped query proof required.
- Tenant/managed-environment isolation: PASS; managed-environment scope remains enforced.
- Run observability: N/A; no new OperationRun.
- OperationRun start UX: N/A.
- Ops-UX lifecycle/summary counts: no changes to lifecycle; current `operation_runs` JSONB posture is inspected.
- Data minimization: PASS; sensitive payloads are not dumped.
- Test governance: PASS; PostgreSQL and focused browser proof are explicit.
- Proportionality: PASS; storage conversion only, no new product truth.
- No premature abstraction: PASS; no new framework.
- Persisted truth: PASS; no new persisted entity/table.
- Behavioral state: PASS; no new state.
- UI semantics: PASS; no UI semantics added.
- Shared pattern first: PASS; existing models/services/tests are reused.
- Provider boundary: PASS; provider payload keys unchanged.
- V1 explicitness / few layers: PASS.
- Spec discipline / bloat check: PASS; one coherent data-layer hardening package.
- Product Surface Contract Gate: PASS as no rendered surface change plus focused regression proof.
## Test Governance Check
- **Test purpose / classification by changed surface**: PostgreSQL migration/type proof, feature/domain regression, focused browser smoke.
- **Affected validation lanes**: pgsql, confidence, browser.
- **Why this lane mix is the narrowest sufficient proof**: storage type is PostgreSQL-specific and payload-backed pages require rendered regression proof, but no full browser audit is needed.
- **Narrowest proving command(s)**: focused Spec 405 pgsql/feature/browser tests once created.
- **Fixture / helper / factory / seed / context cost risks**: minimal explicit fixtures only; no shared default broadening.
- **Expensive defaults or shared helper growth introduced?**: no planned.
- **Heavy-family additions, promotions, or visibility changes**: focused browser proof only.
- **Surface-class relief / special coverage rule**: no UI code changes; browser is regression proof.
- **Closing validation and reviewer handoff**: verify inventory coverage, migration proof, no sensitive payload output, no speculative indexes, and final gate result.
- **Budget / baseline / trend follow-up**: record if pgsql/browser runtime materially increases.
- **Review-stop questions**: Is every `json` column classified? Is every conversion justified? Does every new index have existing query proof? Did any UI or product behavior change? Was staging-like validation completed or properly conditioned?
- **Escalation path**: document-in-feature for bounded findings; follow-up-spec for unsafe conversion, online migration need, or unresolved product/schema decision.
- **Active feature PR close-out entry**: Guardrail / Database Hardening.
- **Why no dedicated follow-up spec is needed**: the slice is bounded unless implementation finds large-table online migration or unresolved product/schema ownership.
## Implementation Phases
### Phase 1 - Inventory and Classification
Build schema inventory from PostgreSQL, migrations, models, casts, factories, query paths, and tests. Produce the draft inventory matrix before any migration.
### Phase 2 - Conversion and Index Design
Write focused migration(s) only for `CONVERT` columns. Add no index unless tied to an existing query path. Keep rollback explicit.
### Phase 3 - Regression and Preservation Proof
Add PostgreSQL and feature tests proving column type, semantic preservation, read/write behavior, scope/authorization, and representative domain behavior.
### Phase 4 - Browser, Staging, and Report
Run focused browser proof, staging-like validation where available, final dirty-state checks, and complete the implementation report with gate result.
## Complexity Tracking
| Violation | Why Needed | Simpler Alternative Rejected Because |
|-----------|------------|--------------------------------------|
| None planned | N/A | N/A |
## Proportionality Review
- **Current operator problem**: trust-layer payload storage is inconsistent and can undermine future evidence/report/lifecycle work.
- **Existing structure is insufficient because**: older `json` payload columns conflict with current PostgreSQL/jsonb posture and query/index expectations.
- **Narrowest correct implementation**: classify all columns, convert only selected existing columns, add only query-backed indexes, and prove behavior.
- **Ownership cost created**: migration review, PostgreSQL tests, focused browser proof, and implementation report.
- **Alternative intentionally rejected**: blanket conversion plus blanket indexes.
- **Release truth**: current-release data-layer readiness.

441
specs/405-json-to-jsonb-data-layer-hardening/spec.md Normal file
View File

@ -0,0 +1,441 @@
# Feature Specification: Spec 405 - JSON-to-JSONB Data-layer Hardening
**Feature Branch**: `405-json-to-jsonb-data-layer-hardening`
**Created**: 2026-06-23
**Status**: Draft / Ready for implementation preparation review
**Type**: Database hardening / migration proof / data integrity spec
**Runtime posture**: Data-layer conversion and proof only. No new product behavior, no new product concepts, and no new rendered UI surfaces.
**Input**: User-provided "Spec 405 - JSON-to-JSONB Data-layer Hardening" draft from `/Users/ahmeddarrazi/.codex/attachments/b7628b1e-f536-40b3-8364-0f6f476c59ac/pasted-text.txt`, current repo truth, Spec 400 product-contract audit context, Specs 401-404 implementation/proof context, roadmap/spec-candidate queue, and live PostgreSQL schema inspection through Laravel Boost.
## Candidate Selection Context
- **Selected candidate**: JSON-to-JSONB Data-layer Hardening.
- **Source location**: Direct user-provided Spec 405 draft, promoted from the Spec 400 P1 data-layer concern that important structured payload columns still use PostgreSQL `json` even though the project standard prefers `jsonb` for queryable policy, evidence, backup, restore, report, provider, and audit payloads.
- **Why selected**: `docs/product/spec-candidates.md` currently reports no safe automatic next-best-prep target, but the operator supplied a concrete manual candidate. The live PostgreSQL schema confirms active `json` columns remain in trust-layer tables such as `policy_versions`, `backup_items`, `restore_runs`, `audit_logs`, `alert_deliveries`, `managed_environment_permissions`, settings tables, and provider/environment metadata paths, while newer foundations already use `jsonb`.
- **Roadmap relationship**: Supports current governance and architecture hardening by reducing data-layer ambiguity before governance artifact lifecycle/retention, report-output expansion, or broader runtime/browser audit work proceeds.
- **Close alternatives deferred**:
- `governance-artifact-lifecycle-retention-runtime`: broader P2 lifecycle semantics; must not be hidden inside a storage-type conversion.
- `provider-readiness-onboarding-productization`: optional UX/productization work; unrelated to payload storage type proof.
- `cross-domain-indicator-runtime-follow-through`: semantic runtime adoption; not a database hardening gate.
- `manual-system-panel-browser-fixture-or-audit-procedure`: validation procedure work; not a payload storage hardening slice.
- Reopening Specs 400-404 or Specs 378-404: forbidden. They are read-only context and proof lineage only.
- **Completed-spec guardrail result**:
- No `specs/405-json-to-jsonb-data-layer-hardening/` package existed before this preparation.
- A local and remote branch named `405-dach-trust-datenschutz-security-website-surface` existed before preparation, but no TenantPilot `specs/405-*` package existed. The operator explicitly supplied Spec ID 405, so this package uses the requested number and records the branch-prefix collision as preparation context.
- Specs 400, 401, 402, 403, and 404 contain audit, implementation-report, validation, browser, or completed-task signals and are read-only historical context. Their close-out notes, validation results, completed task markers, screenshots, smoke history, and implementation reports must not be rewritten, normalized, unchecked, or removed.
- **Smallest viable implementation slice**: Inventory every PostgreSQL `json` and `jsonb` column, classify every `json` column, convert only appropriate queryable trust-layer `json` columns to `jsonb`, add only query-backed indexes, prove semantic payload preservation and existing behavior, run PostgreSQL migration/rollback validation, run focused regression tests and representative browser proof for payload-backed surfaces, and record a final JSON/JSONB inventory matrix plus gate result.
- **Feature description for Spec Kit**: Prove and harden TenantPilot structured payload storage by inventorying all PostgreSQL `json` and `jsonb` columns, converting only appropriate queryable trust-layer `json` columns to `jsonb`, preserving payload semantics and scope boundaries, and documenting local plus staging-like validation without adding new product behavior or UI surfaces.
## Spec Candidate Check *(mandatory - SPEC-GATE-001)*
- **Problem**: TenantPilot stores critical structured proof and operational metadata in a mix of PostgreSQL `json` and `jsonb`; important older payload columns are still `json` even though the project standard prefers `jsonb` for queryable snapshots, backup/restore payloads, evidence/report metadata, and audit context.
- **Today's failure**: Future governance, evidence, report, backup, restore, and lifecycle work can build on inconsistent storage assumptions, miss query/index opportunities, or let agents guess which payloads are queryable/trust-layer data versus low-risk configuration values.
- **User-visible improvement**: Operators and reviewers get stronger confidence that evidence, backup, restore, report, provider, audit, and OperationRun-adjacent payloads remain semantically preserved while becoming query-ready where the product already depends on structured payload proof.
- **Smallest enterprise-capable version**: A schema inventory, explicit classification matrix, targeted `json` to `jsonb` migrations with rollback notes, query-backed indexes only, semantic preservation tests, PostgreSQL lane validation, focused browser proof for representative payload-backed surfaces, and a spec-local implementation report.
- **Explicit non-goals**: No new evidence semantics, provider state model, backup/restore feature, governance lifecycle/retention/export/delete/hold behavior, report template, UI surface, navigation, authorization model, broad normalization, event-sourcing redesign, or completed-spec rewrite.
- **Permanent complexity imported**: One spec package, one future implementation report/inventory matrix, targeted migrations, focused tests, and possibly justified indexes. No new persisted entity/table, enum/status family, runtime abstraction, product vocabulary, UI framework, or provider framework is approved.
- **Why now**: Spec 400 identified the data-layer gap as a P1 condition, Specs 401-404 closed adjacent action, authorization, evidence/currentness, and management-report runtime proof gates, and the live schema still shows older `json` payload columns in trust-layer areas.
- **Why not local**: Converting one column locally would not prove all payload columns were classified, excluded columns were intentional, indexes were query-backed, and report/evidence/backup/restore/provider/audit behavior remained unchanged.
- **Approval class**: Core Enterprise.
- **Red flags triggered**: Foundation/hardening language and many tables. Defense: the slice is bounded to existing storage types, existing payload semantics, and proof; it forbids new models, product concepts, broad frameworks, speculative indexes, and UI changes.
- **Score**: Nutzen: 2 | Dringlichkeit: 2 | Scope: 2 | Komplexitaet: 1 | Produktnaehe: 1 | Wiederverwendung: 2 | **Gesamt: 10/12**
- **Decision**: approve as a bounded data-layer hardening and proof package.
## Problem Statement
TenantPilot depends on structured payloads for policy snapshots, backup items, restore previews/results, audit metadata, provider permission/readiness details, evidence summaries, report receipts, review packs, settings, alerts, and OperationRun-adjacent proof. Some of these payloads remain PostgreSQL `json`, while newer code already uses `jsonb` for similar trust-layer data.
This spec answers:
```text
Can TenantPilot safely convert appropriate important json payload columns to jsonb without data loss, product behavior changes, authorization drift, evidence/currentness drift, report/restore/backup regressions, or speculative indexing?
```
A conversion is successful only when every `json` column was intentionally classified, selected conversions preserve semantic payload content, excluded columns have explicit reasons, migrations and rollback behavior are documented, PostgreSQL validation passes, and existing trust-layer behavior remains unchanged.
## Product / Business Value
- Reduces future data-layer ambiguity around queryable governance, evidence, backup, restore, provider, report, and audit payloads.
- Makes later lifecycle/retention/export/reporting work smaller and safer by establishing clear payload storage posture first.
- Prevents false confidence from application casts alone; storage type, query paths, indexes, tests, and staging-like validation must agree.
- Keeps trust-layer payload semantics stable while hardening the substrate.
## Primary Users / Operators
- Product owner and release reviewer deciding whether the platform is ready for lifecycle/retention and customer-output expansion.
- Engineering reviewer validating migration safety, rollback posture, and test coverage.
- MSP/operator relying on backup, restore, evidence, provider, report, and audit proof.
- Support/platform operator diagnosing payload-backed behavior without raw sensitive payload leakage.
## Spec Scope Fields *(mandatory)*
- **Scope**: database schema inventory, classification, migrations, indexes, tests, validation, focused browser proof, and final data-layer implementation report.
- **Primary Routes / Surfaces**: No route or UI surface is added or changed. Focused browser proof later inspects representative existing payload-backed surfaces only, such as Evidence Overview, Monitoring -> Operations, Provider readiness/detail, Backup/Restore proof, Review Pack or Stored Report output.
- **Data Ownership**:
- Workspace-owned payloads remain workspace scoped.
- Managed-environment-owned payloads remain workspace plus managed-environment scoped.
- Customer-safe report/review payloads remain bound to existing `ReviewPack`, `StoredReport`, `EnvironmentReview`, and evidence ownership rules.
- No new persisted entity/table/artifact is introduced.
- **RBAC**:
- No authorization behavior changes are allowed.
- Existing policies, gates, customer-output gates, deny-as-not-found behavior, and global search posture must continue to pass.
- Any payload query changed during implementation must preserve workspace and managed-environment scoping.
For canonical-view or mixed-scope surfaces:
- **Default filter behavior when environment-context is active**: unchanged. Payload storage conversion must not alter route-owned workspace/environment context or explicit page filters.
- **Explicit entitlement checks preventing cross-tenant leakage**: any changed JSON key query must retain existing scoped query predicates before or with the payload predicate; tests must cover changed high-risk query paths.
## No Legacy / No Backward Compatibility Constraint *(mandatory)*
TenantPilot is pre-production for this data-layer hardening.
- **Compatibility posture**: canonical storage hardening with current payload semantics preserved.
- **Legacy aliases, fallback readers, hidden routes, duplicate UI, old labels, or historical fixtures kept?**: no new compatibility aliases, dual-write paths, fallback readers, hidden routes, duplicate UI, or legacy fixtures are allowed unless this spec is updated with an explicit exception.
- **Why clean replacement is safe now**: PostgreSQL `json` columns already contain valid JSON. Conversion to `jsonb` is a storage hardening change, and semantic behavior must be proven through decoded JSON equality/key checks rather than raw string order.
## UI Surface Impact *(mandatory - UI-COV-001)*
Does this spec add, remove, rename, or materially change any reachable UI surface?
- [x] No UI surface impact
- [ ] Existing page changed
- [ ] New page/route added
- [ ] Navigation changed
- [ ] Filament panel/provider surface changed
- [ ] New modal/drawer/wizard/action added
- [ ] New table/form/state added
- [ ] Customer-facing surface changed
- [ ] Dangerous action changed
- [ ] Status/evidence/review presentation changed
- [ ] Workspace/environment context presentation changed
## UI/Productization Coverage
N/A - no reachable UI surface impact.
- **No-impact rationale**: This spec changes only storage type, migrations, proof, and tests. It may require focused browser proof to show existing payload-backed pages still render correctly after conversion, but it must not edit Filament resources/pages/widgets, Livewire components, Blade views, CSS, JavaScript, navigation, routes, modals, actions, tables, forms, or rendered copy.
## Product Surface Impact *(mandatory for UI-affecting specs; otherwise N/A)*
Reference: `docs/product/standards/product-surface-contract.md`.
- **Product Surface Contract applies?**: no rendered product surface is changed. The contract is used only as a regression lens because payload conversion supports evidence, provider, restore, backup, report, and OperationRun proof surfaces.
- **Page archetype**: N/A for implementation changes. Browser proof later names inspected existing archetypes.
- **Primary user question**: N/A for runtime UI; regression question is "Do existing payload-backed surfaces still render truthful state after conversion?"
- **Primary action**: N/A.
- **Surface budget result**: N/A.
- **Technical Annex / deep-link demotion**: unchanged. Raw payloads, IDs, OperationRun links, evidence deep links, source keys, and provider payloads must not become more visible.
- **Canonical status vocabulary**: unchanged.
- **Visible complexity impact**: neutral.
- **Product Surface exceptions**: none.
## Browser Verification Plan *(mandatory)*
- **Browser proof required?**: yes during implementation, as backend regression proof for existing payload-backed surfaces.
- **No-browser rationale**: N/A for implementation because conversion affects storage used by existing rendered trust surfaces, even though no UI file is changed.
- **Focused path when required**:
1. Evidence overview or evidence anchor surface renders the expected current/stale/missing proof state.
2. Monitoring -> Operations or OperationRun detail renders summary/context/failure proof correctly.
3. Provider detail/readiness/freshness surface renders provider metadata/permission state correctly.
4. Backup or restore proof surface renders preview/result/payload-backed state correctly.
5. Review Pack, Stored Report, or report receipt surface renders evidence/report payload state correctly.
- **Primary interaction to execute**: read existing pages, inspect payload-backed status and links, verify no 500, Livewire, Filament, console, or network errors, and verify no raw/internal payload exposure.
- **Console, Livewire, Filament, network, and 500-error checks**: required for focused browser proof.
- **Full-suite failure triage**: unrelated browser failures may be documented only when the focused Spec 405 path is green and evidence supports the classification.
## Human Product Sanity Check *(mandatory)*
- **Required?**: yes for final implementation report, focused on unchanged trust semantics rather than new UI design.
- **No-human-sanity rationale**: no new rendered surface, but human sanity must confirm the change did not make existing payload-backed surfaces claim stronger proof, expose raw data, or confuse current/released/failed/partial states.
- **Reviewer questions**: Are payload semantics unchanged? Are proof/currentness states still truthful? Are technical details still demoted? Are authorization/customer-safe boundaries unchanged? Does the final report avoid overclaiming staging/production readiness?
- **Planned result location**: `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`.
## Product Surface Merge Gate Checklist *(mandatory)*
- [x] No-legacy posture or approved exception recorded.
- [x] Product Surface Impact is `N/A - no rendered product surface changed` with rationale.
- [x] Browser proof is planned as focused regression proof for existing payload-backed surfaces.
- [x] Human Product Sanity is planned for unchanged trust semantics.
- [x] Product Surface exceptions are `none`.
- [x] Implementation report will state tests/browser result, Livewire v4 compliance, provider registration location, global search posture, destructive/high-impact action posture, asset strategy, deployment impact, visible complexity outcome, and no completed-spec rewrite assertion.
## Cross-Cutting / Shared Pattern Reuse
- **Cross-cutting feature?**: yes at the data layer; no runtime interaction family is implemented.
- **Interaction class(es)**: evidence/report viewers, OperationRun proof, provider readiness, backup/restore proof, audit metadata, alert delivery payloads, and settings payloads are storage consumers only.
- **Systems touched**: database migrations, model casts only where required, query paths only where required, tests, and implementation report.
- **Existing pattern(s) to extend**: Laravel migrations, PostgreSQL information schema inspection, existing model casts, existing scoped query paths, existing Pest PostgreSQL lane, existing browser smoke patterns.
- **Shared contract / presenter / builder / renderer to reuse**: N/A - no UI/runtime interaction contract is added.
- **Why the existing shared path is sufficient or insufficient**: Existing payload consumers are sufficient; the spec hardens storage beneath them. Any missing query path or cast defect must be fixed locally, not by creating a new abstraction.
- **Allowed deviation and why**: none planned.
- **Consistency impact**: payload meaning, scope, authorization, report/evidence truth, and audit metadata must stay consistent before and after conversion.
- **Review focus**: no speculative indexes, no data loss, no raw payload leakage, no broad normalization, no customer-output semantics change, and no completed-spec rewrite.
## OperationRun UX Impact
- **Touches OperationRun start/completion/link UX?**: no UX change. Existing `operation_runs` columns are already `jsonb` in the live schema, but OperationRun-adjacent proof paths must be inspected and regression-tested.
- **Shared OperationRun UX contract/layer reused**: N/A.
- **Delegated start/completion UX behaviors**: N/A.
- **Local surface-owned behavior that remains**: N/A.
- **Queued DB-notification policy**: N/A.
- **Terminal notification path**: N/A.
- **Exception required?**: none.
## Provider Boundary / Platform Core Check
- **Shared provider/platform boundary touched?**: yes at storage inspection level only.
- **Boundary classification**: mixed storage consumers; provider-specific payload semantics remain provider-owned, while storage and scope rules are platform-core.
- **Seams affected**: provider connection metadata, managed-environment permission details, policy snapshots/versions, backup/restore payloads, report/evidence payloads, and audit metadata.
- **Neutral platform terms preserved or introduced**: payload, metadata, evidence, operation, workspace, managed environment, provider connection, report, backup, restore.
- **Provider-specific semantics retained and why**: Microsoft/Intune keys inside existing payloads remain unchanged because the spec does not rename or reinterpret payload keys.
- **Why this does not deepen provider coupling accidentally**: storage type conversion does not introduce provider-specific platform contracts, labels, or taxonomies.
- **Follow-up path**: broader provider readiness/productization remains a separate manual candidate if later evidence requires it.
## UI / Surface Guardrail Impact
N/A - no operator-facing surface change. Focused browser proof later verifies existing surfaces still render after backend conversion.
## Decision-First Surface Role
N/A - no surface changed.
## Audience-Aware Disclosure
N/A - no disclosure changed. Tests and browser proof must still confirm raw/internal payloads do not become default-visible or customer-visible because of conversion or query changes.
## UI/UX Surface Classification
N/A - no surface changed.
## Operator Surface Contract
N/A - no new or materially refactored operator-facing page.
## Proportionality Review *(mandatory when structural complexity is introduced)*
- **New source of truth?**: no.
- **New persisted entity/table/artifact?**: no new entity/table. Migrations may alter storage type of existing columns only.
- **New abstraction?**: no.
- **New enum/state/reason family?**: no.
- **New cross-domain UI framework/taxonomy?**: no.
- **Current operator problem**: trust-layer payload storage is inconsistent and makes later evidence/report/lifecycle work riskier.
- **Existing structure is insufficient because**: leaving older high-risk payloads as `json` conflicts with the repo's PostgreSQL/jsonb posture and makes query/index proof ambiguous.
- **Narrowest correct implementation**: classify all JSON columns, convert only selected existing columns, add only query-backed indexes, and prove semantic preservation.
- **Ownership cost**: migration review, rollback notes, PostgreSQL lane tests, focused regression/browser proof, and final inventory/report maintenance inside the feature.
- **Alternative intentionally rejected**: blanket conversion of every `json` column and blanket GIN indexes. Both would import unnecessary migration risk and write overhead without product proof.
- **Release truth**: current-release data-layer hardening required before broader lifecycle/reporting work.
## Testing / Lane / Runtime Impact *(mandatory for runtime behavior changes)*
- **Test purpose / classification**: Feature, PostgreSQL, and Browser regression. Unit tests only for pure helpers if implementation introduces a bounded helper for semantic comparison or schema inspection.
- **Validation lane(s)**: PostgreSQL lane for migrations/jsonb/indexes; confidence/feature lane for payload behavior; focused browser lane for rendered regression proof.
- **Why this classification and these lanes are sufficient**: SQLite cannot prove PostgreSQL `jsonb`, GIN/expression indexes, or type conversion. Feature tests prove Laravel casts and behavior; browser proof proves representative existing pages still render.
- **New or expanded test families**: one focused Spec 405 test set only; avoid broad browser audit and heavy governance expansion.
- **Fixture / helper cost impact**: minimal explicit fixtures for selected converted columns; no global seed/default context broadening.
- **Heavy-family visibility / justification**: browser proof is explicit because payload storage backs critical trust surfaces; it must stay focused and not become a full UI audit.
- **Special surface test profile**: backend data-layer hardening with existing rendered trust-surface smoke.
- **Standard-native relief or required special coverage**: no Filament component changes; browser proof is regression-only.
- **Reviewer handoff**: verify every converted column appears in the inventory matrix, every excluded `json` column has a reason, every new index has query proof, and no test or fixture leaks sensitive payloads.
- **Budget / baseline / trend impact**: possible PostgreSQL lane runtime increase; document if material.
- **Escalation needed**: document-in-feature unless a conversion is unsafe or requires product/schema decision, then follow-up-spec.
- **Active feature PR close-out entry**: Guardrail / database hardening.
- **Planned validation commands**:
- `git status --short --branch`
- `git diff --check`
- `cd apps/platform && ./vendor/bin/sail artisan migrate`
- `cd apps/platform && ./vendor/bin/sail artisan migrate:rollback --step=1` only in a disposable local/test database when safe
- `cd apps/platform && ./vendor/bin/sail php vendor/bin/pest -c phpunit.pgsql.xml --filter=Spec405`
- targeted feature tests for evidence, OperationRun, provider, backup, restore, review/report payload behavior
- focused browser smoke for representative payload-backed surfaces
## User Scenarios & Testing *(mandatory)*
### User Story 1 - Inventory and classify structured payload columns (Priority: P1)
As an engineering reviewer, I need a complete inventory and classification of all `json` and `jsonb` columns so I can verify that storage decisions are intentional rather than guessed from column names.
**Why this priority**: No migration should be written until every `json` column has a reasoned decision.
**Independent Test**: Run the information-schema query and compare it to the implementation report matrix; every live `json` or `jsonb` column appears once with action, reason, risk, model/cast, query usage, row count, null/default details, and validation requirement.
**Acceptance Scenarios**:
1. **Given** the live PostgreSQL schema contains `json` and `jsonb` columns, **When** the implementation report is reviewed, **Then** every column is present with one of `CONVERT`, `KEEP_JSON`, `ALREADY_JSONB`, `DEPRECATED`, or `DECISION_REQUIRED`.
2. **Given** a high-risk payload column remains `json`, **When** the report is reviewed, **Then** it has an explicit reason and risk classification rather than being omitted.
### User Story 2 - Convert selected trust-layer payloads safely (Priority: P1)
As an operator and release reviewer, I need selected important payload columns converted to `jsonb` without semantic data loss so evidence, backup, restore, provider, report, and audit behavior remains trustworthy.
**Why this priority**: Conversion is the core hardening value and must be proven before follow-up lifecycle or reporting work depends on queryable payloads.
**Independent Test**: Run the migration against PostgreSQL, compare semantic JSON content before/after for selected samples, verify column types, null/default preservation, and rollback notes.
**Acceptance Scenarios**:
1. **Given** selected columns contain representative payloads, **When** the migration runs, **Then** decoded JSON content and required keys are preserved after conversion to `jsonb`.
2. **Given** a column has nulls or a default, **When** the migration runs, **Then** nullable/default behavior remains unchanged unless the implementation report records a spec-approved reason.
3. **Given** a large or unclear table cannot be safely converted directly, **When** classification is completed, **Then** the column is marked `DECISION_REQUIRED`; the final gate may be `PASS WITH CONDITIONS`, but the column is not forced through a speculative migration.
### User Story 3 - Prove existing behavior and report readiness honestly (Priority: P2)
As a product owner, I need a final implementation report that proves application behavior did not regress and clearly states whether data-layer readiness is `PASS`, `PASS WITH CONDITIONS`, or `FAIL`.
**Why this priority**: Storage conversion is not complete unless runtime behavior, authorization/scope boundaries, and representative rendered surfaces still work.
**Independent Test**: Run targeted feature/PostgreSQL tests, focused browser proof, and final dirty-state checks; verify the implementation report records commands, results, remaining findings, and staging-like validation status.
**Acceptance Scenarios**:
1. **Given** converted payloads support evidence/provider/backup/restore/report/audit flows, **When** targeted tests and browser proof run, **Then** existing behavior remains unchanged and no raw/internal payload becomes customer-visible.
2. **Given** staging/Dokploy validation is unavailable, **When** the final report is written, **Then** production data-layer readiness is no stronger than `PASS WITH CONDITIONS`.
### Edge Cases
- `jsonb` normalizes object key order; tests must compare decoded semantic content, not raw JSON strings.
- Duplicate JSON object keys cannot be reconstructed after `jsonb` normalization; rollback notes must record this limitation.
- Null payloads must remain null where null was the prior contract.
- Empty object/array defaults must remain unchanged where defaults already exist.
- Columns with no model, no current query usage, or unclear product ownership must not be converted silently.
- Indexes must not be added for hypothetical future dashboards, lifecycle work, or export queries.
- Sensitive payload samples must not be printed in reports, logs, screenshots, or committed fixtures.
- Changed JSON key queries must preserve workspace/managed-environment scoping before or with payload predicates.
## Requirements *(mandatory)*
### Functional Requirements
- **FR-405-001**: Implementation MUST inventory every PostgreSQL `json` and `jsonb` column from the live schema before writing migrations.
- **FR-405-002**: The inventory matrix MUST include table, column, current type, row count, nullable state, default, model, payload purpose, existing queries, existing casts, recommended action, reason, and risk.
- **FR-405-003**: Every `json` column MUST be classified as `CONVERT`, `KEEP_JSON`, `DEPRECATED`, or `DECISION_REQUIRED`; every `jsonb` column MUST be classified as `ALREADY_JSONB`.
- **FR-405-004**: Implementation MUST convert only columns classified `CONVERT`.
- **FR-405-005**: High-risk trust-layer payloads MUST be converted or explicitly justified, including policy snapshots/metadata, backup payloads, restore preview/results/metadata, audit metadata, provider permission/readiness details, report/review/evidence payloads, alert delivery payloads, and relevant governance/finding metadata.
- **FR-405-006**: Implementation MUST preserve payload semantic content using decoded JSON equality, key-existence checks, required field checks, or domain-specific state assertions.
- **FR-405-007**: Implementation MUST preserve nullability, defaults, constraints, row counts, and application write/read behavior unless the spec is updated with an explicit exception.
- **FR-405-008**: Implementation MUST document rollback behavior and limitations, including `jsonb` key-order normalization and duplicate-key normalization.
- **FR-405-009**: Implementation MUST add indexes only for existing query paths and MUST document table/column/key, query path, reason, expected benefit, write-overhead risk, and validation proof.
- **FR-405-010**: Implementation MUST NOT add speculative GIN, expression, or partial indexes for future dashboards, lifecycle work, export work, or "might need later" reasoning.
- **FR-405-011**: Implementation MUST update model casts only where required to preserve existing Laravel behavior after conversion.
- **FR-405-012**: Implementation MUST fix code paths that assumed textual JSON representation only when tests prove the assumption breaks after `jsonb` conversion.
- **FR-405-013**: Implementation MUST not rename payload keys or change payload meaning.
- **FR-405-014**: Implementation MUST not add new product features, UI surfaces, navigation, authorization models, provider semantics, governance lifecycle behavior, report templates, or normalized replacement tables.
- **FR-405-015**: Implementation MUST prove evidence/currentness behavior from Spec 403 does not regress for converted evidence/report/review payloads.
- **FR-405-016**: Implementation MUST prove authorization and customer-safe boundaries from Specs 401 and 402 do not regress for any changed payload query path.
- **FR-405-017**: Implementation MUST prove management-report PDF/runtime behavior from Spec 404 does not regress where stored report/review payloads or report metadata are in scope.
- **FR-405-018**: Implementation MUST run PostgreSQL-specific migration/type assertions for converted columns.
- **FR-405-019**: Implementation MUST run targeted read/write tests for converted payloads.
- **FR-405-020**: Implementation MUST run focused browser proof for representative payload-backed surfaces.
- **FR-405-021**: Implementation MUST produce `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md` using the required report structure.
- **FR-405-022**: Implementation MUST record final gate result as `PASS`, `PASS WITH CONDITIONS`, or `FAIL`.
- **FR-405-023**: Implementation MUST not claim production data-layer readiness if staging-like PostgreSQL validation is unavailable or incomplete.
- **FR-405-024**: Implementation MUST preserve completed historical spec artifacts as read-only context.
### Non-Functional Requirements
- **NFR-405-001**: Migrations MUST be safe, incremental, reversible where practical, and explicit about lock/runtime risk.
- **NFR-405-002**: PostgreSQL-specific behavior MUST be validated in PostgreSQL, not SQLite.
- **NFR-405-003**: Reports, logs, screenshots, and test output MUST avoid secrets, tokens, raw credential payloads, and sensitive raw provider/customer payloads.
- **NFR-405-004**: The implementation MUST avoid widening test fixtures, browser setup, workspace/member context, or provider setup beyond the narrow proof needs.
- **NFR-405-005**: Any remaining P0/P1 data-layer proof gap MUST be reported, not silently deferred.
## Key Entities *(include if feature involves data)*
- **JSON/JSONB Column Inventory Row**: Spec-local report row representing a live schema column, its current type, ownership, usage, action, reason, and risk.
- **Conversion Migration**: One or more Laravel migrations that alter selected existing columns from `json` to `jsonb` using PostgreSQL-safe conversion.
- **Index Justification Row**: Spec-local report row proving a new JSONB index is tied to an existing query path and bounded overhead.
- **Data Validation Row**: Spec-local report row proving row counts, null counts, type changes, and semantic payload checks before and after conversion.
## Success Criteria *(mandatory)*
### Measurable Outcomes
- **SC-405-001**: 100% of live PostgreSQL `json` and `jsonb` columns appear in the inventory matrix.
- **SC-405-002**: 100% of `json` columns have an explicit action and reason.
- **SC-405-003**: 100% of converted columns have type assertions proving `jsonb` after migration.
- **SC-405-004**: 100% of converted columns have semantic preservation proof for representative non-sensitive sample payloads or an explicit no-row/no-sample note.
- **SC-405-005**: 100% of added JSONB indexes have existing query-path justification and validation proof.
- **SC-405-006**: Targeted evidence, OperationRun, provider, backup, restore, review/report, authorization/scope, and customer-safe regression tests pass or produce documented bounded findings.
- **SC-405-007**: Focused browser proof covers at least five representative payload-backed surfaces or documents exact unavailable paths and resulting gate condition.
- **SC-405-008**: Final implementation report records gate result, validation commands, staging-like validation status, remaining findings, and recommended next step.
## Initial Repo Truth Snapshot
Live PostgreSQL schema inspection on 2026-06-23 found these active `json` columns requiring classification:
```text
alert_deliveries.payload
alert_rules.tenant_allowlist
audit_logs.metadata
backup_items.assignments
backup_items.metadata
backup_items.payload
backup_schedules.days_of_week
backup_schedules.policy_types
backup_sets.metadata
managed_environment_onboarding_sessions.state
managed_environment_permissions.details
managed_environments.metadata
managed_environments.rbac_canary_results
managed_environments.rbac_last_warnings
policies.metadata
policy_versions.assignments
policy_versions.metadata
policy_versions.scope_tags
policy_versions.secret_fingerprints
policy_versions.snapshot
restore_runs.group_mapping
restore_runs.metadata
restore_runs.preview
restore_runs.requested_items
restore_runs.results
tenant_settings.value
workspace_settings.value
```
The same inspection found several newer trust-layer columns already using `jsonb`, including baseline, evidence, finding, review pack, review publication resolution, provider connection, stored report, support request, notification, and operation run payload columns. Implementation must still verify casts and query paths for `ALREADY_JSONB` rows where relevant.
## Required Final Report Structure
Implementation MUST create `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md` with these sections:
1. Candidate Gate Result.
2. Scope Confirmation.
3. Dirty State.
4. JSON/JSONB Inventory Matrix.
5. Migrations Added.
6. Index Justification.
7. Runtime Changes Made.
8. Tests Added or Updated.
9. Data Validation.
10. Browser Proof.
11. Regression Proof.
12. Staging Validation.
13. Remaining Findings.
14. Deferred Items.
15. Validation Commands.
16. Product Surface, Filament, Deployment, and Recommended Next Step Close-Out, including tests/browser result, Livewire v4 compliance, provider registration location, global search posture, destructive/high-impact action posture, asset strategy, deployment impact, visible complexity outcome, no completed-spec rewrite assertion, and recommended next step.
## Gate Rules
- **PASS**: no P0/P1 findings remain; high-risk `json` columns are converted or justified; local and staging-like PostgreSQL validation pass; payload semantic preservation, targeted tests, and representative browser proof pass.
- **PASS WITH CONDITIONS**: no P0 remains; local/test migration proof is strong; remaining P1 conditions such as staging validation or rollback proof are bounded and documented.
- **FAIL**: any P0 remains; payload data is lost; migration cannot safely run; application behavior regresses; evidence/currentness or authorization/customer-safe boundaries regress; or high-risk `json` columns remain unconverted without justification.
## Follow-up Spec Candidates
- Governance Artifact Lifecycle & Retention runtime, only after data-layer readiness is `PASS` or conditions do not affect lifecycle storage.
- Bounded online migration strategy if any large table cannot safely use direct `ALTER COLUMN ... TYPE jsonb`.
- Provider readiness/onboarding productization if browser/regression proof reveals provider-facing user friction unrelated to storage type.
- Full browser/runtime audit remains separate and must not be folded into this spec.
## Assumptions
- PostgreSQL is the authoritative local/staging database for this proof.
- The product is still pre-production under the constitution, so compatibility shims are not required unless explicitly approved.
- Existing payload keys and product meanings are correct unless tests reveal a defect; this spec does not reinterpret payload schemas.
- Staging/Dokploy validation may be externally blocked; if so, final readiness must be conditional.
## Open Questions
- None blocking preparation. Implementation must decide column actions from live schema inventory and repo usage evidence, not from this preparation draft alone.

157
specs/405-json-to-jsonb-data-layer-hardening/tasks.md Normal file
View File

@ -0,0 +1,157 @@
# Tasks: Spec 405 - JSON-to-JSONB Data-layer Hardening
**Input**: `specs/405-json-to-jsonb-data-layer-hardening/spec.md`, `plan.md`, `checklists/requirements.md`, user-provided Spec 405 draft, Spec 400 P1 data-layer risk, related Specs 401-404 proof context, live PostgreSQL schema inspection, and repo truth.
**Tests**: Required. This is a runtime data-layer change. Use Pest 4, PostgreSQL lane for JSONB/migration/index proof, focused feature tests for domain behavior, and focused browser proof for existing payload-backed rendered surfaces.
## Test Governance Checklist
- [x] Lane assignment is `PostgreSQL + Feature + focused Browser`, with no full browser/runtime audit claim.
- [x] New or changed tests stay in the smallest honest family; browser coverage is explicit and focused.
- [x] Shared helpers, factories, seeds, fixtures, workspace/member context, and provider setup stay cheap by default or explicitly opt in.
- [x] Planned validation commands cover the change without broad unrelated suite cost.
- [x] Browser proof is required for representative existing payload-backed surfaces even though no UI file changes.
- [x] Human Product Sanity confirms unchanged trust semantics and no raw payload exposure.
- [x] Any material budget, baseline, trend, or escalation note is recorded in `implementation-report.md`.
## Phase 1: Preparation And Safety
**Purpose**: Establish repo safety, read the active package, and prevent scope drift.
- [x] T001 Read `specs/405-json-to-jsonb-data-layer-hardening/spec.md`, `plan.md`, `tasks.md`, and `checklists/requirements.md`.
- [x] T002 Record current branch, HEAD, dirty state, tracked changed files, untracked files, and `git diff --check` in `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`.
- [x] T003 Re-read `AGENTS.md`, `.specify/memory/constitution.md`, `docs/ai-coding-rules.md`, `docs/architecture-guidelines.md`, `docs/performance-guidelines.md`, `docs/security-guidelines.md`, `docs/testing-guidelines.md`, and `docs/product/standards/product-surface-contract.md`.
- [x] T004 Re-read Specs 400, 401, 402, 403, and 404 implementation reports as read-only context and record their gate results/conditions in `implementation-report.md`.
- [x] T005 Confirm no UI surfaces, routes, navigation, product concepts, authorization model, lifecycle semantics, provider semantics, normalized replacement tables, or completed specs will be edited.
- [x] T006 Create `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md` with sections A through P from `spec.md`.
## Phase 2: Schema Inventory And Usage Mapping
**Purpose**: Prove every JSON column decision is intentional before migration work begins.
- [x] T007 Query PostgreSQL `information_schema.columns` for all live `json` and `jsonb` columns and add each row to the JSON/JSONB Inventory Matrix in `implementation-report.md`.
- [x] T008 For every inventory row, add row count, null count, nullable state, default, indexes, constraints, foreign-key context where relevant, table-size signal, lock/runtime-risk assessment, and direct-vs-online migration decision.
- [x] T009 [P] Map model casts and model ownership for alert, audit, backup, restore, policy, provider/environment, settings, review/report, evidence, finding, and OperationRun-related rows in `apps/platform/app/Models/`.
- [x] T010 [P] Map factory and fixture usage for inventoried columns under `apps/platform/database/factories/` and `apps/platform/tests/`.
- [x] T011 [P] Map query usage for JSON paths, `whereJson*`, `->` key predicates, raw SQL JSON expressions, and JSONB index candidates under `apps/platform/app/` and `apps/platform/tests/`.
- [x] T012 [P] Map rendered/regression consumers for evidence, OperationRun, provider, backup, restore, review pack, stored report, audit, and alert payloads under `apps/platform/app/Filament/` and `apps/platform/resources/views/`.
- [x] T013 Classify each inventory row as `CONVERT`, `KEEP_JSON`, `ALREADY_JSONB`, `DEPRECATED`, or `DECISION_REQUIRED` with risk `P0`, `P1`, `P2`, `P3`, or `None`.
- [x] T014 Stop before migrations if any live `json` column lacks classification, reason, and risk.
## Phase 3: User Story 1 - Inventory and classify structured payload columns (Priority: P1)
**Goal**: Deliver a complete schema and usage matrix that lets reviewers verify every storage decision.
**Independent Test**: The inventory matrix includes every live PostgreSQL `json` and `jsonb` column exactly once and every `json` column has an explicit action/reason/risk.
### Tests for User Story 1
- [x] T015 [P] [US1] Add a PostgreSQL schema inventory assertion for live JSON/JSONB columns in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`.
- [x] T016 [P] [US1] Add a report-coverage assertion or manual checklist entry ensuring every live `json` column is represented in `specs/405-json-to-jsonb-data-layer-hardening/implementation-report.md`.
### Implementation for User Story 1
- [x] T017 [US1] Complete implementation report section D with the JSON/JSONB Inventory Matrix.
- [x] T018 [US1] Mark high-risk columns for policy snapshots, backup payloads, restore previews/results, audit metadata, provider permission/readiness details, review/report payloads, and alert delivery payloads as `CONVERT` or explicitly justified non-conversions.
- [x] T019 [US1] Mark unclear ownership/product-decision columns as `DECISION_REQUIRED` instead of converting by assumption.
- [x] T020 [US1] Record existing `ALREADY_JSONB` trust-layer columns so implementation reviewers can verify consistency and avoid duplicate work.
## Phase 4: User Story 2 - Convert selected trust-layer payloads safely (Priority: P1)
**Goal**: Convert selected existing `json` payload columns to `jsonb` while preserving semantic payload content and behavior.
**Independent Test**: Migrations run on PostgreSQL, selected columns become `jsonb`, semantic payload checks pass, null/default behavior is preserved, and rollback limitations are documented.
### Tests for User Story 2
- [x] T021 [P] [US2] Add PostgreSQL migration/type assertions for every converted column in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`.
- [x] T022 [P] [US2] Add semantic payload preservation tests with non-sensitive samples for policy, backup, restore, audit, provider/environment, settings, and alert/report payload categories in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`.
- [x] T023 [P] [US2] Add model read/write tests for converted columns whose casts or factories require explicit proof in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`.
- [x] T024 [P] [US2] Add query-path tests for any changed JSON key query or new JSONB index in `apps/platform/tests/Feature/Database/JsonbDataLayerHardeningTest.php`.
### Implementation for User Story 2
- [x] T025 [US2] Create one or more focused migrations under `apps/platform/database/migrations/` converting only `CONVERT` columns to `jsonb`.
- [x] T026 [US2] Preserve nullability, defaults, and constraints in each conversion migration.
- [x] T027 [US2] Add rollback SQL or explicit rollback limitations to each conversion migration.
- [x] T028 [US2] Add justified JSONB indexes only where an existing query path is documented in `implementation-report.md`.
- [x] T029 [US2] Update model casts in `apps/platform/app/Models/` only where tests prove current behavior needs a cast adjustment.
- [x] T030 [US2] Update query code only where tests prove textual JSON assumptions break after conversion.
- [x] T031 [US2] Update factories or fixtures only where required by the new targeted tests and avoid broad seed/default changes.
- [x] T032 [US2] Complete implementation report sections E, F, G, and I with migrations, indexes, runtime changes, and data validation.
## Phase 5: User Story 3 - Prove existing behavior and report readiness honestly (Priority: P2)
**Goal**: Prove the conversion did not regress existing trust-layer behavior and produce a gate result.
**Independent Test**: Targeted domain tests, PostgreSQL tests, focused browser proof, and staging-like validation status are recorded; final report uses `PASS`, `PASS WITH CONDITIONS`, or `FAIL` honestly.
### Tests for User Story 3
- [x] T033 [P] [US3] Prove evidence/currentness regression paths with existing focused browser smoke and Spec405 storage tests; no converted evidence/report/review columns required new feature tests.
- [x] T034 [P] [US3] Add OperationRun/audit regression proof for converted audit and OperationRun-adjacent payload paths in the focused Spec405 test plus browser proof.
- [x] T035 [P] [US3] Add provider readiness/freshness/permission regression proof for converted provider/environment payload paths in the focused Spec405 test plus browser proof.
- [x] T036 [P] [US3] Add backup payload regression proof in the focused Spec405 test plus browser proof.
- [x] T037 [P] [US3] Add restore preview/result payload regression proof in the focused Spec405 test plus browser proof.
- [x] T038 [P] [US3] Prove review pack, stored report, management-report PDF/runtime, and customer-output regression paths with existing focused browser smoke; those columns were already `jsonb`.
- [x] T039 [P] [US3] Confirm no authorization/scope query path changed beyond the JSONB-safe `BackupItem` scope; covered by targeted tests and no RBAC edits.
- [x] T040 [US3] Run focused existing browser smoke tests for representative existing payload-backed surfaces; no new UI smoke artifact was needed because no rendered surface changed.
### Implementation for User Story 3
- [x] T041 [US3] Browser-proof Evidence Overview or evidence anchor rendering after conversion and record route, actor, workspace/environment, expected state, actual state, and console/runtime result.
- [x] T042 [US3] Browser-proof Monitoring -> Operations or OperationRun detail payload-backed rendering after conversion.
- [x] T043 [US3] Browser-proof provider detail/readiness/freshness payload-backed rendering after conversion.
- [x] T044 [US3] Browser-proof backup or restore payload-backed proof rendering after conversion.
- [x] T045 [US3] Browser-proof Review Pack, Stored Report, management-report PDF/runtime, or report receipt payload-backed rendering after conversion.
- [x] T046 [US3] Record Human Product Sanity result in `implementation-report.md`: no stronger proof claims, no raw payload exposure, unchanged current/released/failed/partial semantics.
- [x] T047 [US3] Run local PostgreSQL migration validation and record migration success, row counts, null counts, converted types, and sample semantic checks in `implementation-report.md`.
- [x] T048 [US3] Run rollback validation in a disposable local/test database where safe, or record exact rollback limitation and risk.
- [x] T049 [US3] Run staging-like PostgreSQL validation if accessible and record migration/app boot/tests/browser/payload-read-write results.
- [x] T050 [US3] If staging/Dokploy validation is not accessible, record the blocker and ensure final gate result is no stronger than `PASS WITH CONDITIONS`.
- [x] T051 [US3] Complete implementation report sections H, J, K, L, M, N, O, and P.
## Phase 6: Final Validation And Close-Out
**Purpose**: Confirm the package is ready for review and no unrelated work entered the slice.
- [x] T052 Run `git diff --check` from repo root and record result in `implementation-report.md`.
- [x] T053 Run project formatting for changed PHP files according to repo convention and record result.
- [x] T054 Run focused PostgreSQL Spec405 tests and record exact command/results.
- [x] T055 Run targeted feature/domain regression tests and record exact command/results.
- [x] T056 Run focused browser proof and record exact command/results, or exact blocker without claiming browser proof.
- [x] T057 Verify reports, logs, screenshots, and fixtures do not include secrets, tokens, raw credential payloads, sensitive raw provider payloads, or customer-sensitive raw payloads.
- [x] T058 Record final dirty state, tracked changed files, and untracked files in `implementation-report.md`.
- [x] T059 Confirm no completed historical specs were rewritten or stripped of close-out/validation/task/smoke/browser history.
- [x] T060 Confirm `implementation-report.md` and the final response state Livewire v4 compliance, provider registration location, global search posture, destructive/high-impact action posture, asset strategy, tests/browser result, deployment impact, visible complexity outcome, no completed-spec rewrite assertion, and explicit application implementation status.
## Non-Goals Checklist
- [x] NT001 Do not add product features, report templates, navigation, UI surfaces, actions, modals, forms, tables, or customer output.
- [x] NT002 Do not change authorization model, roles, capabilities, policies, or global search posture except to fix a direct regression caused by this conversion and already covered by the spec.
- [x] NT003 Do not add governance lifecycle, retention, export, delete, hold, or artifact-state semantics.
- [x] NT004 Do not add new persisted entities/tables, normalized replacement tables, status families, taxonomies, provider frameworks, or broad abstractions.
- [x] NT005 Do not add speculative indexes.
- [x] NT006 Do not rename payload keys or reinterpret payload meaning.
- [x] NT007 Do not print or commit sensitive raw payload samples.
- [x] NT008 Do not rewrite completed specs or remove historical implementation evidence.
## Dependencies And Execution Order
- Phase 1 must complete before any migration or runtime edit.
- Phase 2 inventory and classification must complete before Phase 4 conversion.
- User Story 1 is the required MVP and blocks migration implementation.
- User Story 2 depends on completed classification and creates the conversion.
- User Story 3 depends on converted columns and proves behavior/readiness.
- Final validation depends on all in-scope tests, browser proof, and implementation report completion.
## Parallel Execution Examples
- After T007/T008, T009 through T012 can run in parallel because they inspect different surfaces.
- T021 through T024 can be drafted in parallel after conversion decisions are known.
- T033 through T039 can run in parallel by domain test family after migrations exist.
- T041 through T045 can run as separate focused browser paths if fixture setup is independent.
## Recommended Implementation Strategy
Deliver User Story 1 first and stop if the inventory is incomplete or reveals unresolved product/schema decisions. Then convert the smallest justified group of high-risk trust-layer columns, prove semantic preservation, and add only the narrow tests/browser proof required for the changed columns. Treat staging validation as a release gate: if unavailable, report `PASS WITH CONDITIONS`, not full readiness.