TenantAtlas/apps/platform/tests/Support/TestLaneBudget.php
Ahmed Darrazi 97262787c9
Some checks failed
PR Fast Feedback / fast-feedback (pull_request) Failing after 4m15s
feat: implement runtime trend recalibration reporting
2026-04-18 09:34:23 +02:00

625 lines
26 KiB
PHP

<?php
declare(strict_types=1);
namespace Tests\Support;
use InvalidArgumentException;
final class TestLaneBudget
{
public function __construct(
public readonly int $thresholdSeconds,
public readonly string $baselineSource,
public readonly string $enforcement,
public readonly string $lifecycleState,
public readonly ?int $baselineDeltaTargetPercent = null,
public readonly ?string $notes = null,
public readonly ?string $reviewCadence = null,
) {
}
/**
* @param array<string, mixed> $budget
*/
public static function fromArray(array $budget): self
{
if (! isset($budget['thresholdSeconds'], $budget['baselineSource'], $budget['enforcement'], $budget['lifecycleState'])) {
throw new InvalidArgumentException('Budget declarations must define thresholdSeconds, baselineSource, enforcement, and lifecycleState.');
}
return new self(
thresholdSeconds: (int) $budget['thresholdSeconds'],
baselineSource: (string) $budget['baselineSource'],
enforcement: (string) $budget['enforcement'],
lifecycleState: (string) $budget['lifecycleState'],
baselineDeltaTargetPercent: isset($budget['baselineDeltaTargetPercent']) ? (int) $budget['baselineDeltaTargetPercent'] : null,
notes: isset($budget['notes']) ? (string) $budget['notes'] : null,
reviewCadence: isset($budget['reviewCadence']) ? (string) $budget['reviewCadence'] : null,
);
}
/**
* @return array<string, int|float|string>
*/
public function evaluate(float $measuredSeconds): array
{
$budgetStatus = 'within-budget';
if ($measuredSeconds > $this->thresholdSeconds) {
$budgetStatus = in_array($this->enforcement, ['report-only', 'warn'], true)
? 'warning'
: 'over-budget';
}
return array_filter([
'thresholdSeconds' => $this->thresholdSeconds,
'baselineSource' => $this->baselineSource,
'enforcement' => $this->enforcement,
'lifecycleState' => $this->lifecycleState,
'baselineDeltaTargetPercent' => $this->baselineDeltaTargetPercent,
'measuredSeconds' => round($measuredSeconds, 6),
'budgetStatus' => $budgetStatus,
'notes' => $this->notes,
'reviewCadence' => $this->reviewCadence,
], static fn (mixed $value): bool => $value !== null);
}
/**
* @param array<string, mixed> $budgetTarget
* @return array<string, int|float|string>
*/
public static function evaluateBudgetTarget(array $budgetTarget, float $measuredSeconds): array
{
if (! isset($budgetTarget['targetType'], $budgetTarget['targetId'])) {
throw new InvalidArgumentException('Budget targets must define targetType and targetId.');
}
$evaluation = self::fromArray($budgetTarget)->evaluate($measuredSeconds);
return array_merge([
'budgetId' => (string) ($budgetTarget['budgetId'] ?? sprintf('%s-%s', $budgetTarget['targetType'], $budgetTarget['targetId'])),
'targetType' => (string) $budgetTarget['targetType'],
'targetId' => (string) $budgetTarget['targetId'],
], $evaluation);
}
/**
* @param list<array<string, mixed>> $budgetTargets
* @param array<string, float> $classificationTotals
* @param array<string, float> $familyTotals
* @return list<array<string, int|float|string>>
*/
public static function evaluateBudgetTargets(
array $budgetTargets,
float $laneSeconds,
array $classificationTotals,
array $familyTotals,
): array {
$evaluations = [];
foreach ($budgetTargets as $budgetTarget) {
$targetType = (string) ($budgetTarget['targetType'] ?? '');
$targetId = (string) ($budgetTarget['targetId'] ?? '');
if ($targetType === '' || $targetId === '') {
continue;
}
$measuredSeconds = match ($targetType) {
'lane' => $laneSeconds,
'classification' => (float) ($classificationTotals[$targetId] ?? 0.0),
'family' => (float) ($familyTotals[$targetId] ?? 0.0),
default => 0.0,
};
$evaluations[] = self::evaluateBudgetTarget($budgetTarget, $measuredSeconds);
}
return $evaluations;
}
/**
* @param array<string, mixed> $contract
* @return array<string, int|float|string>
*/
public static function evaluateGovernanceContract(array $contract, float $measuredSeconds): array
{
foreach (['laneId', 'summaryThresholdSeconds', 'evaluationThresholdSeconds', 'normalizedThresholdSeconds'] as $requiredKey) {
if (! array_key_exists($requiredKey, $contract)) {
throw new InvalidArgumentException(sprintf('Governance contracts must define [%s].', $requiredKey));
}
}
$normalizedThresholdSeconds = (float) $contract['normalizedThresholdSeconds'];
if ($normalizedThresholdSeconds <= 0) {
throw new InvalidArgumentException('Governance contracts must define a positive normalizedThresholdSeconds value.');
}
$enforcementLevel = (string) ($contract['enforcementLevel'] ?? 'warn');
$budgetStatus = 'within-budget';
if ($measuredSeconds > $normalizedThresholdSeconds) {
$budgetStatus = in_array($enforcementLevel, ['report-only', 'warn'], true)
? 'warning'
: 'over-budget';
}
return array_filter([
'laneId' => (string) $contract['laneId'],
'summaryThresholdSeconds' => (float) $contract['summaryThresholdSeconds'],
'evaluationThresholdSeconds' => (float) $contract['evaluationThresholdSeconds'],
'normalizedThresholdSeconds' => $normalizedThresholdSeconds,
'baselineSource' => (string) ($contract['baselineSource'] ?? 'measured-lane'),
'enforcementLevel' => $enforcementLevel,
'lifecycleState' => (string) ($contract['lifecycleState'] ?? 'draft'),
'decisionStatus' => (string) ($contract['decisionStatus'] ?? 'pending'),
'measuredSeconds' => round($measuredSeconds, 6),
'budgetStatus' => $budgetStatus,
'reconciliationRationale' => isset($contract['reconciliationRationale']) ? (string) $contract['reconciliationRationale'] : null,
], static fn (mixed $value): bool => $value !== null);
}
/**
* @return list<array<string, int|float|string>>
*/
public static function enforcementProfiles(): array
{
$fastFeedbackBudget = TestLaneManifest::lane('fast-feedback')['budget'];
$confidenceBudget = TestLaneManifest::lane('confidence')['budget'];
$browserBudget = TestLaneManifest::lane('browser')['budget'];
$heavyGovernanceContract = TestLaneManifest::heavyGovernanceBudgetContract();
return [
[
'policyId' => 'pr-fast-feedback-budget',
'laneId' => 'fast-feedback',
'triggerClass' => 'pull-request',
'thresholdSource' => 'lane-budget',
'baseThresholdSeconds' => (int) $fastFeedbackBudget['thresholdSeconds'],
'varianceAllowanceSeconds' => 15,
'effectiveThresholdSeconds' => (int) $fastFeedbackBudget['thresholdSeconds'] + 15,
'enforcementMode' => 'hard-fail',
'lifecycleState' => (string) $fastFeedbackBudget['lifecycleState'],
'reviewCadence' => 'revisit after two stable CI pull request runs',
],
[
'policyId' => 'main-confidence-budget',
'laneId' => 'confidence',
'triggerClass' => 'mainline-push',
'thresholdSource' => 'lane-budget',
'baseThresholdSeconds' => (int) $confidenceBudget['thresholdSeconds'],
'varianceAllowanceSeconds' => 30,
'effectiveThresholdSeconds' => (int) $confidenceBudget['thresholdSeconds'] + 30,
'enforcementMode' => 'soft-warn',
'lifecycleState' => (string) $confidenceBudget['lifecycleState'],
'reviewCadence' => 'tighten after two stable dev runs',
],
[
'policyId' => 'heavy-governance-manual-budget',
'laneId' => 'heavy-governance',
'triggerClass' => 'manual',
'thresholdSource' => 'governance-contract',
'baseThresholdSeconds' => (int) round((float) $heavyGovernanceContract['normalizedThresholdSeconds']),
'varianceAllowanceSeconds' => 15,
'effectiveThresholdSeconds' => (int) round((float) $heavyGovernanceContract['normalizedThresholdSeconds']) + 15,
'enforcementMode' => 'soft-warn',
'lifecycleState' => (string) $heavyGovernanceContract['lifecycleState'],
'reviewCadence' => 'manual heavy validation must stabilize before schedule enablement',
],
[
'policyId' => 'heavy-governance-scheduled-budget',
'laneId' => 'heavy-governance',
'triggerClass' => 'scheduled',
'thresholdSource' => 'governance-contract',
'baseThresholdSeconds' => (int) round((float) $heavyGovernanceContract['normalizedThresholdSeconds']),
'varianceAllowanceSeconds' => 15,
'effectiveThresholdSeconds' => (int) round((float) $heavyGovernanceContract['normalizedThresholdSeconds']) + 15,
'enforcementMode' => 'trend-only',
'lifecycleState' => (string) $heavyGovernanceContract['lifecycleState'],
'reviewCadence' => 'convert from trend-only only after stable scheduled evidence exists',
],
[
'policyId' => 'browser-manual-budget',
'laneId' => 'browser',
'triggerClass' => 'manual',
'thresholdSource' => 'lane-budget',
'baseThresholdSeconds' => (int) $browserBudget['thresholdSeconds'],
'varianceAllowanceSeconds' => 20,
'effectiveThresholdSeconds' => (int) $browserBudget['thresholdSeconds'] + 20,
'enforcementMode' => 'soft-warn',
'lifecycleState' => (string) $browserBudget['lifecycleState'],
'reviewCadence' => 'tighten after two stable browser validation runs',
],
[
'policyId' => 'browser-scheduled-budget',
'laneId' => 'browser',
'triggerClass' => 'scheduled',
'thresholdSource' => 'lane-budget',
'baseThresholdSeconds' => (int) $browserBudget['thresholdSeconds'],
'varianceAllowanceSeconds' => 20,
'effectiveThresholdSeconds' => (int) $browserBudget['thresholdSeconds'] + 20,
'enforcementMode' => 'trend-only',
'lifecycleState' => (string) $browserBudget['lifecycleState'],
'reviewCadence' => 'convert from trend-only only after stable scheduled evidence exists',
],
];
}
/**
* @return array<string, int|float|string>
*/
public static function enforcementProfile(string $laneId, string $triggerClass): array
{
foreach (self::enforcementProfiles() as $profile) {
if ($profile['laneId'] === $laneId && $profile['triggerClass'] === $triggerClass) {
return $profile;
}
}
throw new InvalidArgumentException(sprintf('Unknown trigger-aware budget profile for lane [%s] and trigger [%s].', $laneId, $triggerClass));
}
/**
* @param array<string, int|float|string> $profile
* @return array<string, int|float|string|null>
*/
public static function evaluateTriggerAwareBudgetProfile(array $profile, float $measuredSeconds): array
{
foreach (['laneId', 'triggerClass', 'baseThresholdSeconds', 'varianceAllowanceSeconds', 'effectiveThresholdSeconds', 'enforcementMode', 'lifecycleState'] as $requiredKey) {
if (! array_key_exists($requiredKey, $profile)) {
throw new InvalidArgumentException(sprintf('Trigger-aware budget profiles must define [%s].', $requiredKey));
}
}
$baseThresholdSeconds = (float) $profile['baseThresholdSeconds'];
$effectiveThresholdSeconds = (float) $profile['effectiveThresholdSeconds'];
$budgetStatus = 'within-budget';
if ($measuredSeconds > $effectiveThresholdSeconds) {
$budgetStatus = 'over-budget';
} elseif ($measuredSeconds > $baseThresholdSeconds) {
$budgetStatus = 'warning';
}
$blockingStatus = match ((string) $profile['enforcementMode']) {
'hard-fail' => match ($budgetStatus) {
'over-budget' => 'blocking',
'warning' => 'non-blocking-warning',
default => 'informational',
},
'soft-warn' => $budgetStatus === 'within-budget' ? 'informational' : 'non-blocking-warning',
'trend-only' => 'informational',
default => throw new InvalidArgumentException(sprintf('Unknown enforcement mode [%s].', $profile['enforcementMode'])),
};
return array_merge($profile, [
'measuredSeconds' => round($measuredSeconds, 6),
'budgetStatus' => $budgetStatus,
'blockingStatus' => $blockingStatus,
'primaryFailureClassId' => $budgetStatus === 'within-budget' ? null : 'budget-breach',
]);
}
/**
* @return array<string, int|float|string|null>
*/
public static function evaluateLaneForTrigger(string $laneId, string $triggerClass, float $measuredSeconds): array
{
return self::evaluateTriggerAwareBudgetProfile(
self::enforcementProfile($laneId, $triggerClass),
$measuredSeconds,
);
}
/**
* @param array<string, mixed> $baselineSnapshot
* @param array<string, mixed> $currentSnapshot
* @return array<string, float>
*/
public static function compareSnapshots(array $baselineSnapshot, array $currentSnapshot): array
{
$baselineSeconds = round((float) ($baselineSnapshot['wallClockSeconds'] ?? 0.0), 6);
$currentSeconds = round((float) ($currentSnapshot['wallClockSeconds'] ?? 0.0), 6);
$deltaSeconds = round($currentSeconds - $baselineSeconds, 6);
$deltaPercent = $baselineSeconds > 0.0
? round(($deltaSeconds / $baselineSeconds) * 100, 6)
: 0.0;
return [
'baselineSeconds' => $baselineSeconds,
'currentSeconds' => $currentSeconds,
'deltaSeconds' => $deltaSeconds,
'deltaPercent' => $deltaPercent,
];
}
/**
* @param array<string, mixed> $contract
* @param array<string, mixed> $baselineSnapshot
* @param array<string, mixed> $currentSnapshot
* @param list<string> $remainingOpenFamilies
* @param list<string> $followUpDebt
* @return array<string, mixed>
*/
public static function buildOutcomeRecord(
array $contract,
array $baselineSnapshot,
array $currentSnapshot,
array $remainingOpenFamilies,
string $justification,
array $followUpDebt = [],
): array {
$comparison = self::compareSnapshots($baselineSnapshot, $currentSnapshot);
return array_filter([
'outcomeId' => sprintf('%s-final-outcome', (string) ($contract['laneId'] ?? 'heavy-governance')),
'decisionStatus' => (string) ($contract['decisionStatus'] ?? 'pending'),
'finalThresholdSeconds' => round((float) ($contract['normalizedThresholdSeconds'] ?? 0.0), 6),
'finalMeasuredSeconds' => $comparison['currentSeconds'],
'deltaSeconds' => $comparison['deltaSeconds'],
'deltaPercent' => $comparison['deltaPercent'],
'remainingOpenFamilies' => array_values($remainingOpenFamilies),
'justification' => $justification,
'followUpDebt' => $followUpDebt !== [] ? array_values($followUpDebt) : null,
], static fn (mixed $value): bool => $value !== null);
}
public static function trendVarianceFloorSeconds(string $laneId, ?string $triggerClass = null): int
{
$matchingProfiles = array_values(array_filter(
self::enforcementProfiles(),
static fn (array $profile): bool => $profile['laneId'] === $laneId
&& ($triggerClass === null || $profile['triggerClass'] === $triggerClass),
));
if ($matchingProfiles === [] && $triggerClass !== null) {
$matchingProfiles = array_values(array_filter(
self::enforcementProfiles(),
static fn (array $profile): bool => $profile['laneId'] === $laneId,
));
}
if ($matchingProfiles !== []) {
return (int) max(array_map(
static fn (array $profile): int => (int) ($profile['varianceAllowanceSeconds'] ?? 0),
$matchingProfiles,
));
}
return match ($laneId) {
'junit' => 30,
'profiling' => 45,
default => 15,
};
}
public static function nearBudgetHeadroomSeconds(string $laneId): int
{
return match ($laneId) {
'fast-feedback' => 20,
'confidence', 'junit' => 45,
'browser' => 25,
'heavy-governance' => 30,
'profiling' => 120,
default => max(self::trendVarianceFloorSeconds($laneId), 15),
};
}
/**
* @return array<string, mixed>
*/
public static function recalibrationPolicy(string $laneId): array
{
return [
'laneId' => $laneId,
'baselineRequiresExplicitReview' => true,
'budgetRequiresExplicitReview' => true,
'minimumBaselineEvidenceSamples' => 3,
'minimumBudgetEvidenceSamples' => $laneId === 'fast-feedback' ? 4 : 5,
'baselineAllowedRationales' => [
'lane-scope-change',
'infrastructure-shift',
'post-improvement-reset',
'manual-hold',
],
'approvedBaselineRationales' => [
'lane-scope-change',
'infrastructure-shift',
'post-improvement-reset',
],
'budgetAllowedRationales' => [
'infrastructure-shift',
'sustained-erosion',
'manual-hold',
],
'approvedBudgetRationales' => [
'infrastructure-shift',
'sustained-erosion',
],
'rejectedRationales' => [
'noise-rejected',
'manual-hold',
],
];
}
/**
* @param list<array<string, mixed>> $historyRecords
* @return array<string, mixed>
*/
public static function buildRecalibrationDecisionRecord(
string $laneId,
string $targetType,
array $assessment,
array $historyRecords,
string $decisionStatus,
string $rationaleCode,
string $recordedIn,
?float $proposedValueSeconds = null,
?string $notes = null,
): array {
if (! in_array($targetType, ['baseline', 'budget'], true)) {
throw new InvalidArgumentException(sprintf('Unknown recalibration target type [%s].', $targetType));
}
if (! in_array($decisionStatus, ['candidate', 'approved', 'rejected'], true)) {
throw new InvalidArgumentException(sprintf('Unknown recalibration decision status [%s].', $decisionStatus));
}
$policy = self::recalibrationPolicy($laneId);
$minimumEvidenceSamples = $targetType === 'budget'
? (int) $policy['minimumBudgetEvidenceSamples']
: (int) $policy['minimumBaselineEvidenceSamples'];
$minimumEvidenceSamples = $decisionStatus === 'rejected'
? 1
: $minimumEvidenceSamples;
$evidenceRunRefs = array_values(array_filter(array_map(
static fn (array $record): ?string => is_string($record['runRef'] ?? null) && $record['runRef'] !== ''
? (string) $record['runRef']
: null,
$historyRecords,
)));
$evidenceRunRefs = array_slice(array_values(array_unique($evidenceRunRefs)), 0, $minimumEvidenceSamples);
if (count($evidenceRunRefs) < $minimumEvidenceSamples) {
throw new InvalidArgumentException(sprintf(
'Recalibration decisions for [%s] require at least %d evidence samples.',
$targetType,
$minimumEvidenceSamples,
));
}
if ($decisionStatus === 'approved') {
$allowedRationales = $targetType === 'baseline'
? $policy['approvedBaselineRationales']
: $policy['approvedBudgetRationales'];
if (! in_array($rationaleCode, $allowedRationales, true)) {
throw new InvalidArgumentException(sprintf(
'Approved %s recalibration decisions must use one of [%s].',
$targetType,
implode(', ', $allowedRationales),
));
}
} elseif ($decisionStatus === 'rejected') {
if (! in_array($rationaleCode, $policy['rejectedRationales'], true)) {
throw new InvalidArgumentException('Rejected recalibration decisions must use a rejected rationale.');
}
} else {
$allowedRationales = $targetType === 'baseline'
? $policy['baselineAllowedRationales']
: $policy['budgetAllowedRationales'];
if (! in_array($rationaleCode, $allowedRationales, true)) {
throw new InvalidArgumentException(sprintf(
'Candidate %s recalibration decisions must use one of [%s].',
$targetType,
implode(', ', $allowedRationales),
));
}
}
$currentRecord = $historyRecords[0] ?? [];
$previousValueSeconds = $targetType === 'baseline'
? (float) ($currentRecord['baselineSeconds'] ?? $currentRecord['wallClockSeconds'] ?? 0.0)
: (float) ($currentRecord['budgetSeconds'] ?? 0.0);
$defaultNotes = match ($decisionStatus) {
'approved' => sprintf(
'Approved %s recalibration for lane [%s] after reviewing %d comparable samples.',
$targetType,
$laneId,
count($evidenceRunRefs),
),
'rejected' => sprintf(
'Rejected %s recalibration for lane [%s] because current evidence is not strong enough to move repository truth.',
$targetType,
$laneId,
),
default => sprintf(
'Candidate %s recalibration for lane [%s]. Review the active spec or PR before changing repository truth.',
$targetType,
$laneId,
),
};
return [
'targetType' => $targetType,
'decisionStatus' => $decisionStatus,
'evidenceRunRefs' => $evidenceRunRefs,
'previousValueSeconds' => round($previousValueSeconds, 6),
'proposedValueSeconds' => $proposedValueSeconds !== null ? round($proposedValueSeconds, 6) : null,
'rationaleCode' => $rationaleCode,
'recordedIn' => $recordedIn,
'notes' => $notes ?? $defaultNotes,
];
}
/**
* @param list<array<string, mixed>> $historyRecords
* @return list<array<string, mixed>>
*/
public static function automaticRecalibrationDecisions(
string $laneId,
array $assessment,
array $historyRecords,
string $recordedIn,
): array {
$recommendation = (string) ($assessment['recalibrationRecommendation'] ?? 'none');
$windowStatus = (string) ($assessment['windowStatus'] ?? 'stable');
$currentRecord = $historyRecords[0] ?? [];
$decisionRecords = [];
if ($recommendation === 'review-baseline') {
$decisionRecords[] = self::buildRecalibrationDecisionRecord(
laneId: $laneId,
targetType: 'baseline',
assessment: $assessment,
historyRecords: $historyRecords,
decisionStatus: 'candidate',
rationaleCode: 'manual-hold',
recordedIn: $recordedIn,
proposedValueSeconds: isset($currentRecord['wallClockSeconds']) ? (float) $currentRecord['wallClockSeconds'] : null,
notes: 'Candidate baseline review. Confirm lane-scope, infrastructure, or post-improvement evidence before approving any baseline reset.',
);
}
if ($recommendation === 'review-budget') {
$proposedBudgetSeconds = isset($currentRecord['wallClockSeconds'])
? (float) $currentRecord['wallClockSeconds'] + self::nearBudgetHeadroomSeconds($laneId)
: null;
$decisionRecords[] = self::buildRecalibrationDecisionRecord(
laneId: $laneId,
targetType: 'budget',
assessment: $assessment,
historyRecords: $historyRecords,
decisionStatus: 'candidate',
rationaleCode: 'sustained-erosion',
recordedIn: $recordedIn,
proposedValueSeconds: $proposedBudgetSeconds,
notes: 'Candidate budget review. Only approve after sustained erosion is confirmed and the active spec or PR records why the budget should move.',
);
}
if ($decisionRecords === [] && in_array($windowStatus, ['insufficient-history', 'noisy', 'scope-changed'], true)) {
$decisionRecords[] = self::buildRecalibrationDecisionRecord(
laneId: $laneId,
targetType: 'budget',
assessment: $assessment,
historyRecords: $historyRecords,
decisionStatus: 'rejected',
rationaleCode: $windowStatus === 'noisy' ? 'noise-rejected' : 'manual-hold',
recordedIn: $recordedIn,
notes: 'Recalibration is rejected for this cycle because the comparison window is not stable enough to justify moving repository truth.',
);
}
return $decisionRecords;
}
}