ahmido
0d5d1fc9f4
## Summary - add the checked-in Spec 208 heavy-suite classification and family manifest with config-driven lane generation, attribution, and budget reporting - update Pest grouping, guard coverage, wrapper/report contracts, and spec artifacts for the segmented lane model - complete the targeted follow-up pass that re-homes the remaining in-scope confidence hotspots into explicit heavy-governance families ## Acceptance - confidence is repaired and now measures 389.613832s, down from 587.446894s and below the 450s lane budget - confidence is also slightly below the post-Spec-207 baseline of 394.383441s (delta -4.769609s) - this closes the central Spec 208 acceptance issue that had kept the spec open ## Intentionally Re-homed Families - finding-bulk-actions-workflow - drift-bulk-triage-all-matching - baseline-profile-start-surfaces - workspace-settings-slice-management - findings-workflow-surfaces - workspace-only-admin-surface-independence ## Explicit Residual Risk - heavy-governance now measures 318.296962s, above its documented 300s threshold - the cost was not removed; it was moved into the correct lane and made visible on clearly named heavy families - this is documented residual debt, not an open Spec 208 failure ## Validation - focused guard/support validation: 206 passed (3607 assertions) - lane wrapper/report validation completed for confidence and heavy-governance - no full-suite run was performed in this pass by request Co-authored-by: Ahmed Darrazi <ahmed.darrazi@live.de> Reviewed-on: #241
458 lines
18 KiB
PHP
458 lines
18 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Tests\Support;
|
|
|
|
use SimpleXMLElement;
|
|
|
|
final class TestLaneReport
|
|
{
|
|
/**
|
|
* @return array{junit: string, summary: string, budget: string, report: string, profile: string}
|
|
*/
|
|
public static function artifactPaths(string $laneId, ?string $artifactDirectory = null): array
|
|
{
|
|
$directory = trim($artifactDirectory ?? TestLaneManifest::artifactDirectory(), '/');
|
|
|
|
return [
|
|
'junit' => sprintf('%s/%s-latest.junit.xml', $directory, $laneId),
|
|
'summary' => sprintf('%s/%s-latest.summary.md', $directory, $laneId),
|
|
'budget' => sprintf('%s/%s-latest.budget.json', $directory, $laneId),
|
|
'report' => sprintf('%s/%s-latest.report.json', $directory, $laneId),
|
|
'profile' => sprintf('%s/%s-latest.profile.txt', $directory, $laneId),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @return array{slowestEntries: list<array<string, mixed>>, durationsByFile: array<string, float>}
|
|
*/
|
|
public static function parseJUnit(string $filePath, string $laneId): array
|
|
{
|
|
if (! is_file($filePath)) {
|
|
return [
|
|
'slowestEntries' => [],
|
|
'durationsByFile' => [],
|
|
];
|
|
}
|
|
|
|
$useInternalErrors = libxml_use_internal_errors(true);
|
|
$xml = simplexml_load_file($filePath);
|
|
libxml_clear_errors();
|
|
libxml_use_internal_errors($useInternalErrors);
|
|
|
|
if (! $xml instanceof SimpleXMLElement) {
|
|
return [
|
|
'slowestEntries' => [],
|
|
'durationsByFile' => [],
|
|
];
|
|
}
|
|
|
|
$slowestEntries = [];
|
|
$durationsByFile = [];
|
|
|
|
foreach ($xml->xpath('//testcase') ?: [] as $testcase) {
|
|
$rawSubject = trim((string) ($testcase['file'] ?? ''));
|
|
$subject = $rawSubject !== '' ? $rawSubject : trim((string) ($testcase['name'] ?? 'unknown-testcase'));
|
|
$duration = round((float) ($testcase['time'] ?? 0.0), 6);
|
|
$normalizedFile = explode('::', $subject)[0];
|
|
|
|
$slowestEntries[] = [
|
|
'label' => $subject,
|
|
'subject' => $subject,
|
|
'filePath' => $normalizedFile,
|
|
'durationSeconds' => $duration,
|
|
'wallClockSeconds' => $duration,
|
|
'laneId' => $laneId,
|
|
];
|
|
|
|
$durationsByFile[$normalizedFile] = round(($durationsByFile[$normalizedFile] ?? 0.0) + $duration, 6);
|
|
}
|
|
|
|
usort($slowestEntries, static fn (array $left, array $right): int => $right['wallClockSeconds'] <=> $left['wallClockSeconds']);
|
|
|
|
return [
|
|
'slowestEntries' => array_slice($slowestEntries, 0, 10),
|
|
'durationsByFile' => $durationsByFile,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param list<array<string, mixed>> $slowestEntries
|
|
* @param array<string, float> $durationsByFile
|
|
* @return array<string, mixed>
|
|
*/
|
|
public static function buildReport(
|
|
string $laneId,
|
|
float $wallClockSeconds,
|
|
array $slowestEntries,
|
|
array $durationsByFile,
|
|
?string $artifactDirectory = null,
|
|
?string $comparisonProfile = null,
|
|
): array {
|
|
$lane = TestLaneManifest::lane($laneId);
|
|
$laneBudget = TestLaneBudget::fromArray($lane['budget']);
|
|
$laneBudgetEvaluation = $laneBudget->evaluate($wallClockSeconds);
|
|
$artifactPaths = self::artifactPaths($laneId, $artifactDirectory);
|
|
$artifacts = [];
|
|
|
|
foreach ($lane['artifacts'] as $artifactMode) {
|
|
$relativePath = match ($artifactMode) {
|
|
'summary' => $artifactPaths['summary'],
|
|
'junit-xml' => $artifactPaths['junit'],
|
|
'profile-top' => $artifactPaths['profile'],
|
|
'budget-report' => $artifactPaths['budget'],
|
|
default => null,
|
|
};
|
|
|
|
if (! is_string($relativePath)) {
|
|
continue;
|
|
}
|
|
|
|
$artifacts[] = [
|
|
'artifactMode' => $artifactMode,
|
|
'relativePath' => $relativePath,
|
|
'machineReadable' => in_array($artifactMode, ['junit-xml', 'budget-report'], true),
|
|
];
|
|
}
|
|
|
|
$attribution = self::buildAttribution($durationsByFile);
|
|
$budgetEvaluations = TestLaneBudget::evaluateBudgetTargets(
|
|
self::relevantBudgetTargets(
|
|
$laneId,
|
|
$attribution['classificationTotals'],
|
|
$attribution['familyTotals'],
|
|
),
|
|
$wallClockSeconds,
|
|
$attribution['classificationTotals'],
|
|
$attribution['familyTotals'],
|
|
);
|
|
|
|
$enrichedSlowestEntries = array_values(array_map(
|
|
static function (array $entry) use ($attribution): array {
|
|
$filePath = (string) ($entry['filePath'] ?? '');
|
|
$familyId = $attribution['fileToFamily'][$filePath] ?? null;
|
|
$classificationId = $attribution['fileToClassification'][$filePath] ?? null;
|
|
|
|
return array_filter(array_merge($entry, [
|
|
'familyId' => $familyId,
|
|
'classificationId' => $classificationId,
|
|
]), static fn (mixed $value): bool => $value !== null);
|
|
},
|
|
$slowestEntries,
|
|
));
|
|
|
|
$report = [
|
|
'laneId' => $laneId,
|
|
'artifactDirectory' => trim($artifactDirectory ?? TestLaneManifest::artifactDirectory(), '/'),
|
|
'finishedAt' => gmdate('c'),
|
|
'wallClockSeconds' => round($wallClockSeconds, 6),
|
|
'budgetThresholdSeconds' => $laneBudget->thresholdSeconds,
|
|
'budgetBaselineSource' => $laneBudget->baselineSource,
|
|
'budgetEnforcement' => $laneBudget->enforcement,
|
|
'budgetLifecycleState' => $laneBudget->lifecycleState,
|
|
'budgetStatus' => $laneBudgetEvaluation['budgetStatus'],
|
|
'slowestEntries' => $enrichedSlowestEntries,
|
|
'classificationAttribution' => $attribution['classificationAttribution'],
|
|
'familyAttribution' => $attribution['familyAttribution'],
|
|
'budgetEvaluations' => $budgetEvaluations,
|
|
'familyBudgetEvaluations' => array_values(array_filter(
|
|
$budgetEvaluations,
|
|
static fn (array $evaluation): bool => ($evaluation['targetType'] ?? null) === 'family',
|
|
)),
|
|
'artifacts' => $artifacts,
|
|
];
|
|
|
|
if ($laneBudget->baselineDeltaTargetPercent !== null) {
|
|
$report['baselineDeltaTargetPercent'] = $laneBudget->baselineDeltaTargetPercent;
|
|
}
|
|
|
|
$comparison = self::buildSharedFixtureSlimmingComparison($laneId, $wallClockSeconds, $comparisonProfile);
|
|
|
|
if ($comparison !== null) {
|
|
$report['sharedFixtureSlimmingComparison'] = $comparison;
|
|
}
|
|
|
|
return $report;
|
|
}
|
|
|
|
/**
|
|
* @param array<string, mixed> $report
|
|
* @return array{summary: string, budget: string, report: string, profile: string}
|
|
*/
|
|
public static function writeArtifacts(
|
|
string $laneId,
|
|
array $report,
|
|
?string $profileOutput = null,
|
|
?string $artifactDirectory = null,
|
|
): array {
|
|
$artifactPaths = self::artifactPaths($laneId, $artifactDirectory);
|
|
|
|
self::ensureDirectory(dirname(TestLaneManifest::absolutePath($artifactPaths['summary'])));
|
|
|
|
file_put_contents(
|
|
TestLaneManifest::absolutePath($artifactPaths['summary']),
|
|
self::buildSummaryMarkdown($report),
|
|
);
|
|
|
|
file_put_contents(
|
|
TestLaneManifest::absolutePath($artifactPaths['budget']),
|
|
json_encode([
|
|
'laneId' => $report['laneId'],
|
|
'artifactDirectory' => $report['artifactDirectory'],
|
|
'wallClockSeconds' => $report['wallClockSeconds'],
|
|
'budgetThresholdSeconds' => $report['budgetThresholdSeconds'],
|
|
'budgetBaselineSource' => $report['budgetBaselineSource'],
|
|
'budgetEnforcement' => $report['budgetEnforcement'],
|
|
'budgetLifecycleState' => $report['budgetLifecycleState'],
|
|
'budgetStatus' => $report['budgetStatus'],
|
|
'classificationAttribution' => $report['classificationAttribution'],
|
|
'familyAttribution' => $report['familyAttribution'],
|
|
'budgetEvaluations' => $report['budgetEvaluations'],
|
|
'familyBudgetEvaluations' => $report['familyBudgetEvaluations'],
|
|
'sharedFixtureSlimmingComparison' => $report['sharedFixtureSlimmingComparison'] ?? null,
|
|
], JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR),
|
|
);
|
|
|
|
file_put_contents(
|
|
TestLaneManifest::absolutePath($artifactPaths['report']),
|
|
json_encode($report, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR),
|
|
);
|
|
|
|
if (is_string($profileOutput) && trim($profileOutput) !== '') {
|
|
file_put_contents(TestLaneManifest::absolutePath($artifactPaths['profile']), $profileOutput);
|
|
}
|
|
|
|
return $artifactPaths;
|
|
}
|
|
|
|
/**
|
|
* @return array<string, mixed>
|
|
*/
|
|
public static function finalizeLane(
|
|
string $laneId,
|
|
float $wallClockSeconds,
|
|
string $capturedOutput = '',
|
|
?string $comparisonProfile = null,
|
|
): array {
|
|
$artifactPaths = self::artifactPaths($laneId);
|
|
$parsed = self::parseJUnit(TestLaneManifest::absolutePath($artifactPaths['junit']), $laneId);
|
|
$report = self::buildReport(
|
|
laneId: $laneId,
|
|
wallClockSeconds: $wallClockSeconds,
|
|
slowestEntries: $parsed['slowestEntries'],
|
|
durationsByFile: $parsed['durationsByFile'],
|
|
comparisonProfile: $comparisonProfile,
|
|
);
|
|
|
|
self::writeArtifacts($laneId, $report, $capturedOutput);
|
|
|
|
return $report;
|
|
}
|
|
|
|
/**
|
|
* @param array<string, mixed> $report
|
|
*/
|
|
private static function buildSummaryMarkdown(array $report): string
|
|
{
|
|
$lines = [
|
|
'# Test Lane Summary',
|
|
'',
|
|
sprintf('- Lane: %s', $report['laneId']),
|
|
sprintf('- Finished: %s', $report['finishedAt']),
|
|
sprintf('- Wall clock: %.2f seconds', (float) $report['wallClockSeconds']),
|
|
sprintf('- Budget: %d seconds (%s)', (int) $report['budgetThresholdSeconds'], $report['budgetStatus']),
|
|
];
|
|
|
|
if (isset($report['sharedFixtureSlimmingComparison']) && is_array($report['sharedFixtureSlimmingComparison'])) {
|
|
$comparison = $report['sharedFixtureSlimmingComparison'];
|
|
|
|
$lines[] = sprintf(
|
|
'- Shared fixture slimming baseline: %.2f seconds (%s, %+0.2f%%)',
|
|
(float) $comparison['baselineSeconds'],
|
|
(string) $comparison['status'],
|
|
(float) $comparison['deltaPercent'],
|
|
);
|
|
}
|
|
|
|
$lines[] = '';
|
|
$lines[] = '## Slowest entries';
|
|
|
|
foreach ($report['slowestEntries'] as $entry) {
|
|
$label = (string) ($entry['label'] ?? $entry['subject'] ?? 'unknown');
|
|
$lines[] = sprintf('- %s (%.2fs)', $label, (float) ($entry['wallClockSeconds'] ?? $entry['durationSeconds'] ?? 0.0));
|
|
}
|
|
|
|
if (($report['classificationAttribution'] ?? []) !== []) {
|
|
$lines[] = '';
|
|
$lines[] = '## Classification attribution';
|
|
|
|
foreach ($report['classificationAttribution'] as $entry) {
|
|
$lines[] = sprintf('- %s (%.2fs)', $entry['classificationId'], (float) $entry['totalWallClockSeconds']);
|
|
}
|
|
}
|
|
|
|
if (($report['familyAttribution'] ?? []) !== []) {
|
|
$lines[] = '';
|
|
$lines[] = '## Family attribution';
|
|
|
|
foreach ($report['familyAttribution'] as $entry) {
|
|
$lines[] = sprintf('- %s [%s] (%.2fs)', $entry['familyId'], $entry['classificationId'], (float) $entry['totalWallClockSeconds']);
|
|
}
|
|
}
|
|
|
|
return implode(PHP_EOL, $lines).PHP_EOL;
|
|
}
|
|
|
|
/**
|
|
* @return array<string, mixed>
|
|
*/
|
|
private static function buildAttribution(array $durationsByFile): array
|
|
{
|
|
$classificationTotals = [];
|
|
$familyTotals = [];
|
|
$classificationHotspots = [];
|
|
$familyHotspots = [];
|
|
$fileToClassification = [];
|
|
$fileToFamily = [];
|
|
|
|
foreach ($durationsByFile as $filePath => $duration) {
|
|
$family = TestLaneManifest::familyForFile($filePath);
|
|
$mixedResolution = TestLaneManifest::mixedFileResolution($filePath);
|
|
$classificationId = $mixedResolution['primaryClassificationId'] ?? ($family['classificationId'] ?? null);
|
|
|
|
if (! is_string($classificationId) || $classificationId === '') {
|
|
continue;
|
|
}
|
|
|
|
$classificationTotals[$classificationId] = round(($classificationTotals[$classificationId] ?? 0.0) + $duration, 6);
|
|
$classificationHotspots[$classificationId][] = $filePath;
|
|
$fileToClassification[$filePath] = $classificationId;
|
|
|
|
if (is_array($family)) {
|
|
$familyId = (string) $family['familyId'];
|
|
$familyTotals[$familyId] = round(($familyTotals[$familyId] ?? 0.0) + $duration, 6);
|
|
$familyHotspots[$familyId]['classificationId'] = $family['classificationId'];
|
|
$familyHotspots[$familyId]['hotspotFiles'][] = $filePath;
|
|
$fileToFamily[$filePath] = $familyId;
|
|
}
|
|
}
|
|
|
|
$classificationAttribution = array_values(array_map(
|
|
static fn (string $classificationId, float $total): array => [
|
|
'classificationId' => $classificationId,
|
|
'totalWallClockSeconds' => $total,
|
|
'hotspotFiles' => array_values(array_unique($classificationHotspots[$classificationId] ?? [])),
|
|
],
|
|
array_keys($classificationTotals),
|
|
$classificationTotals,
|
|
));
|
|
|
|
usort($classificationAttribution, static fn (array $left, array $right): int => $right['totalWallClockSeconds'] <=> $left['totalWallClockSeconds']);
|
|
|
|
$familyAttribution = array_values(array_map(
|
|
static function (string $familyId, float $total) use ($familyHotspots): array {
|
|
return [
|
|
'familyId' => $familyId,
|
|
'classificationId' => $familyHotspots[$familyId]['classificationId'],
|
|
'totalWallClockSeconds' => $total,
|
|
'hotspotFiles' => array_values(array_unique($familyHotspots[$familyId]['hotspotFiles'] ?? [])),
|
|
];
|
|
},
|
|
array_keys($familyTotals),
|
|
$familyTotals,
|
|
));
|
|
|
|
usort($familyAttribution, static fn (array $left, array $right): int => $right['totalWallClockSeconds'] <=> $left['totalWallClockSeconds']);
|
|
|
|
return [
|
|
'classificationTotals' => $classificationTotals,
|
|
'familyTotals' => $familyTotals,
|
|
'classificationAttribution' => $classificationAttribution,
|
|
'familyAttribution' => $familyAttribution,
|
|
'fileToClassification' => $fileToClassification,
|
|
'fileToFamily' => $fileToFamily,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param array<string, float> $classificationTotals
|
|
* @param array<string, float> $familyTotals
|
|
* @return list<array<string, mixed>>
|
|
*/
|
|
private static function relevantBudgetTargets(
|
|
string $laneId,
|
|
array $classificationTotals,
|
|
array $familyTotals,
|
|
): array {
|
|
return array_values(array_filter(
|
|
TestLaneManifest::budgetTargets(),
|
|
static function (array $budgetTarget) use ($laneId, $classificationTotals, $familyTotals): bool {
|
|
$targetType = (string) ($budgetTarget['targetType'] ?? '');
|
|
$targetId = (string) ($budgetTarget['targetId'] ?? '');
|
|
|
|
return match ($targetType) {
|
|
'lane' => $targetId === $laneId,
|
|
'classification' => array_key_exists($targetId, $classificationTotals),
|
|
'family' => array_key_exists($targetId, $familyTotals),
|
|
default => false,
|
|
};
|
|
},
|
|
));
|
|
}
|
|
|
|
/**
|
|
* @return array<string, float|int|string>|null
|
|
*/
|
|
private static function buildSharedFixtureSlimmingComparison(
|
|
string $laneId,
|
|
float $wallClockSeconds,
|
|
?string $comparisonProfile,
|
|
): ?array {
|
|
if ($comparisonProfile !== 'shared-test-fixture-slimming') {
|
|
return null;
|
|
}
|
|
|
|
$baseline = TestLaneManifest::comparisonBaseline($comparisonProfile, $laneId);
|
|
|
|
if (! is_array($baseline)) {
|
|
return null;
|
|
}
|
|
|
|
$baselineSeconds = (float) $baseline['wallClockSeconds'];
|
|
$deltaSeconds = round($wallClockSeconds - $baselineSeconds, 6);
|
|
$deltaPercent = $baselineSeconds > 0
|
|
? round(($deltaSeconds / $baselineSeconds) * 100, 6)
|
|
: 0.0;
|
|
$targetImprovementPercent = (int) ($baseline['targetImprovementPercent'] ?? 10);
|
|
$maxRegressionPercent = (int) ($baseline['maxRegressionPercent'] ?? 5);
|
|
|
|
$status = 'stable';
|
|
|
|
if ($deltaPercent <= -$targetImprovementPercent) {
|
|
$status = 'improved';
|
|
} elseif ($deltaPercent > $maxRegressionPercent) {
|
|
$status = 'regressed';
|
|
}
|
|
|
|
return [
|
|
'comparisonProfile' => $comparisonProfile,
|
|
'baselineFinishedAt' => (string) $baseline['finishedAt'],
|
|
'baselineSeconds' => $baselineSeconds,
|
|
'deltaSeconds' => $deltaSeconds,
|
|
'deltaPercent' => $deltaPercent,
|
|
'targetImprovementPercent' => $targetImprovementPercent,
|
|
'maxRegressionPercent' => $maxRegressionPercent,
|
|
'status' => $status,
|
|
];
|
|
}
|
|
|
|
private static function ensureDirectory(string $directory): void
|
|
{
|
|
if (is_dir($directory)) {
|
|
return;
|
|
}
|
|
|
|
mkdir($directory, 0777, true);
|
|
}
|
|
} |