ahmido
53e799fea7
## Summary - add Spec 185 workspace recovery posture visibility artifacts under `specs/185-workspace-recovery-posture-visibility` - promote tenant backup health and recovery evidence onto the workspace overview with separate metrics, attention ordering, calmness coverage, and tenant-dashboard drill-throughs - batch visible-tenant backup/recovery derivation to keep the workspace overview query-bounded - align follow-up fixes from the authoritative suite rerun, including dashboard truth-alignment fixtures, canonical backup schedule tenant context, guard-path cleanup, smoke-fixture credential removal, and robust theme asset manifest handling ## Testing - `cd apps/platform && ./vendor/bin/sail bin pint --dirty --format agent` - `cd apps/platform && ./vendor/bin/sail artisan test --compact tests/Unit/Filament/PanelThemeAssetTest.php tests/Feature/Guards/DerivedStateConsumerAdoptionGuardTest.php` - focused regression pack for the previously failing cases passed - full suite JUnit run passed: `3401` tests, `18849` assertions, `0` failures, `0` errors, `8` skips ## Notes - no new schema or persisted workspace recovery model - no provider-registration changes; Filament/Livewire stack remains on Filament v5 and Livewire v4 - no new destructive actions or global search changes Co-authored-by: Ahmed Darrazi <ahmed.darrazi@live.de> Reviewed-on: #216
452 lines
16 KiB
PHP
452 lines
16 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Support\BackupHealth;
|
|
|
|
use App\Models\BackupSchedule;
|
|
use App\Models\BackupSet;
|
|
use App\Models\Tenant;
|
|
use App\Support\BackupQuality\BackupQualityResolver;
|
|
use Carbon\CarbonImmutable;
|
|
use Carbon\CarbonInterface;
|
|
use Illuminate\Support\Arr;
|
|
|
|
final class TenantBackupHealthResolver
|
|
{
|
|
private const POSITIVE_CLAIM_BOUNDARY = 'Backup health reflects backup inputs only and does not prove restore success.';
|
|
|
|
public function __construct(
|
|
private readonly BackupQualityResolver $backupQualityResolver,
|
|
) {}
|
|
|
|
public function assess(Tenant|int $tenant): TenantBackupHealthAssessment
|
|
{
|
|
$tenantId = $tenant instanceof Tenant
|
|
? (int) $tenant->getKey()
|
|
: (int) $tenant;
|
|
|
|
return $this->assessMany([$tenantId])[$tenantId];
|
|
}
|
|
|
|
/**
|
|
* @param iterable<int, Tenant|int> $tenants
|
|
* @return array<int, TenantBackupHealthAssessment>
|
|
*/
|
|
public function assessMany(iterable $tenants): array
|
|
{
|
|
$tenantIds = $this->normalizeTenantIds($tenants);
|
|
|
|
if ($tenantIds === []) {
|
|
return [];
|
|
}
|
|
|
|
$now = CarbonImmutable::now('UTC');
|
|
$latestBackupSets = $this->latestRelevantBackupSets($tenantIds);
|
|
$scheduleFollowUps = $this->scheduleFollowUpEvaluations($tenantIds, $now);
|
|
|
|
$assessments = [];
|
|
|
|
foreach ($tenantIds as $tenantId) {
|
|
$assessments[$tenantId] = $this->assessmentForResolvedInputs(
|
|
tenantId: $tenantId,
|
|
latestBackupSet: $latestBackupSets[$tenantId] ?? null,
|
|
scheduleFollowUp: $scheduleFollowUps[$tenantId] ?? $this->emptyScheduleFollowUpEvaluation(),
|
|
now: $now,
|
|
);
|
|
}
|
|
|
|
return $assessments;
|
|
}
|
|
|
|
private function latestRelevantBackupSet(int $tenantId): ?BackupSet
|
|
{
|
|
return $this->latestRelevantBackupSets([$tenantId])[$tenantId] ?? null;
|
|
}
|
|
|
|
/**
|
|
* @param array<int, int> $tenantIds
|
|
* @return array<int, BackupSet>
|
|
*/
|
|
private function latestRelevantBackupSets(array $tenantIds): array
|
|
{
|
|
if ($tenantIds === []) {
|
|
return [];
|
|
}
|
|
|
|
$latestBackupSetIds = BackupSet::query()
|
|
->withTrashed()
|
|
->whereIn('tenant_id', $tenantIds)
|
|
->whereNotNull('completed_at')
|
|
->orderBy('tenant_id')
|
|
->orderByDesc('completed_at')
|
|
->orderByDesc('id')
|
|
->get([
|
|
'id',
|
|
'tenant_id',
|
|
])
|
|
->unique('tenant_id')
|
|
->pluck('id')
|
|
->all();
|
|
|
|
if ($latestBackupSetIds === []) {
|
|
return [];
|
|
}
|
|
|
|
return BackupSet::query()
|
|
->withTrashed()
|
|
->whereIn('id', $latestBackupSetIds)
|
|
->with([
|
|
'items' => fn ($query) => $query->select([
|
|
'id',
|
|
'tenant_id',
|
|
'backup_set_id',
|
|
'payload',
|
|
'metadata',
|
|
'assignments',
|
|
]),
|
|
])
|
|
->get([
|
|
'id',
|
|
'tenant_id',
|
|
'workspace_id',
|
|
'name',
|
|
'status',
|
|
'item_count',
|
|
'created_by',
|
|
'completed_at',
|
|
'created_at',
|
|
'metadata',
|
|
'deleted_at',
|
|
])
|
|
->keyBy(static fn (BackupSet $backupSet): int => (int) $backupSet->tenant_id)
|
|
->all();
|
|
}
|
|
|
|
private function freshnessEvaluation(?CarbonInterface $latestCompletedAt, CarbonImmutable $now): BackupFreshnessEvaluation
|
|
{
|
|
$cutoffAt = $now->subHours($this->freshnessHours());
|
|
|
|
return new BackupFreshnessEvaluation(
|
|
latestCompletedAt: $latestCompletedAt,
|
|
cutoffAt: $cutoffAt,
|
|
isFresh: $latestCompletedAt?->greaterThanOrEqualTo($cutoffAt) ?? false,
|
|
);
|
|
}
|
|
|
|
private function scheduleFollowUpEvaluation(int $tenantId, CarbonImmutable $now): BackupScheduleFollowUpEvaluation
|
|
{
|
|
return $this->scheduleFollowUpEvaluations([$tenantId], $now)[$tenantId] ?? $this->emptyScheduleFollowUpEvaluation();
|
|
}
|
|
|
|
/**
|
|
* @param array<int, int> $tenantIds
|
|
* @return array<int, BackupScheduleFollowUpEvaluation>
|
|
*/
|
|
private function scheduleFollowUpEvaluations(array $tenantIds, CarbonImmutable $now): array
|
|
{
|
|
if ($tenantIds === []) {
|
|
return [];
|
|
}
|
|
|
|
$schedulesByTenant = BackupSchedule::query()
|
|
->whereIn('tenant_id', $tenantIds)
|
|
->where('is_enabled', true)
|
|
->orderBy('tenant_id')
|
|
->orderBy('next_run_at')
|
|
->orderBy('id')
|
|
->get([
|
|
'id',
|
|
'tenant_id',
|
|
'last_run_status',
|
|
'last_run_at',
|
|
'next_run_at',
|
|
'created_at',
|
|
])
|
|
->groupBy('tenant_id');
|
|
|
|
$evaluations = [];
|
|
|
|
foreach ($tenantIds as $tenantId) {
|
|
$evaluations[$tenantId] = $this->evaluateScheduleFollowUpCollection(
|
|
schedules: $schedulesByTenant->get($tenantId, collect()),
|
|
now: $now,
|
|
);
|
|
}
|
|
|
|
return $evaluations;
|
|
}
|
|
|
|
private function evaluateScheduleFollowUpCollection(iterable $schedules, CarbonImmutable $now): BackupScheduleFollowUpEvaluation
|
|
{
|
|
$graceCutoff = $now->subMinutes($this->scheduleOverdueGraceMinutes());
|
|
|
|
$enabledScheduleCount = 0;
|
|
$overdueScheduleCount = 0;
|
|
$failedRecentRunCount = 0;
|
|
$neverSuccessfulCount = 0;
|
|
$primaryScheduleId = null;
|
|
|
|
foreach ($schedules as $schedule) {
|
|
$enabledScheduleCount++;
|
|
$isOverdue = $schedule->next_run_at?->lessThan($graceCutoff) ?? false;
|
|
$lastRunStatus = strtolower(trim((string) $schedule->last_run_status));
|
|
$needsFollowUpAfterRun = in_array($lastRunStatus, ['failed', 'partial', 'skipped', 'canceled'], true);
|
|
$neverSuccessful = $schedule->last_run_at === null
|
|
&& ($isOverdue || ($schedule->created_at?->lessThan($graceCutoff) ?? false));
|
|
|
|
if ($isOverdue) {
|
|
$overdueScheduleCount++;
|
|
}
|
|
|
|
if ($needsFollowUpAfterRun) {
|
|
$failedRecentRunCount++;
|
|
}
|
|
|
|
if ($neverSuccessful) {
|
|
$neverSuccessfulCount++;
|
|
}
|
|
|
|
if ($primaryScheduleId === null && ($neverSuccessful || $isOverdue || $needsFollowUpAfterRun)) {
|
|
$primaryScheduleId = (int) $schedule->getKey();
|
|
}
|
|
}
|
|
|
|
return new BackupScheduleFollowUpEvaluation(
|
|
hasEnabledSchedules: $enabledScheduleCount > 0,
|
|
enabledScheduleCount: $enabledScheduleCount,
|
|
overdueScheduleCount: $overdueScheduleCount,
|
|
failedRecentRunCount: $failedRecentRunCount,
|
|
neverSuccessfulCount: $neverSuccessfulCount,
|
|
needsFollowUp: $overdueScheduleCount > 0 || $failedRecentRunCount > 0 || $neverSuccessfulCount > 0,
|
|
primaryScheduleId: $primaryScheduleId,
|
|
summaryMessage: $this->scheduleSummaryMessage(
|
|
enabledScheduleCount: $enabledScheduleCount,
|
|
overdueScheduleCount: $overdueScheduleCount,
|
|
failedRecentRunCount: $failedRecentRunCount,
|
|
neverSuccessfulCount: $neverSuccessfulCount,
|
|
),
|
|
);
|
|
}
|
|
|
|
private function emptyScheduleFollowUpEvaluation(): BackupScheduleFollowUpEvaluation
|
|
{
|
|
return new BackupScheduleFollowUpEvaluation(
|
|
hasEnabledSchedules: false,
|
|
enabledScheduleCount: 0,
|
|
overdueScheduleCount: 0,
|
|
failedRecentRunCount: 0,
|
|
neverSuccessfulCount: 0,
|
|
needsFollowUp: false,
|
|
primaryScheduleId: null,
|
|
summaryMessage: null,
|
|
);
|
|
}
|
|
|
|
private function assessmentForResolvedInputs(
|
|
int $tenantId,
|
|
?BackupSet $latestBackupSet,
|
|
BackupScheduleFollowUpEvaluation $scheduleFollowUp,
|
|
CarbonImmutable $now,
|
|
): TenantBackupHealthAssessment {
|
|
$qualitySummary = $latestBackupSet instanceof BackupSet
|
|
? $this->backupQualityResolver->summarizeBackupSet($latestBackupSet)
|
|
: null;
|
|
$freshnessEvaluation = $this->freshnessEvaluation($latestBackupSet?->completed_at, $now);
|
|
|
|
if (! $latestBackupSet instanceof BackupSet) {
|
|
return new TenantBackupHealthAssessment(
|
|
tenantId: $tenantId,
|
|
posture: TenantBackupHealthAssessment::POSTURE_ABSENT,
|
|
primaryReason: TenantBackupHealthAssessment::REASON_NO_BACKUP_BASIS,
|
|
headline: 'No usable backup basis is available.',
|
|
supportingMessage: $this->combineMessages([
|
|
'Create or finish a backup set before relying on restore input.',
|
|
$scheduleFollowUp->summaryMessage,
|
|
]),
|
|
latestRelevantBackupSetId: null,
|
|
latestRelevantCompletedAt: null,
|
|
qualitySummary: null,
|
|
freshnessEvaluation: $freshnessEvaluation,
|
|
scheduleFollowUp: $scheduleFollowUp,
|
|
healthyClaimAllowed: false,
|
|
primaryActionTarget: BackupHealthActionTarget::backupSetsIndex(TenantBackupHealthAssessment::REASON_NO_BACKUP_BASIS),
|
|
positiveClaimBoundary: self::POSITIVE_CLAIM_BOUNDARY,
|
|
);
|
|
}
|
|
|
|
if (! $freshnessEvaluation->isFresh) {
|
|
return new TenantBackupHealthAssessment(
|
|
tenantId: $tenantId,
|
|
posture: TenantBackupHealthAssessment::POSTURE_STALE,
|
|
primaryReason: TenantBackupHealthAssessment::REASON_LATEST_BACKUP_STALE,
|
|
headline: 'Latest backup is stale.',
|
|
supportingMessage: $this->combineMessages([
|
|
$this->latestBackupAgeMessage($latestBackupSet->completed_at, $now),
|
|
$qualitySummary?->hasDegradations() === true ? 'The latest completed backup is also degraded.' : null,
|
|
$scheduleFollowUp->summaryMessage,
|
|
]),
|
|
latestRelevantBackupSetId: (int) $latestBackupSet->getKey(),
|
|
latestRelevantCompletedAt: $latestBackupSet->completed_at,
|
|
qualitySummary: $qualitySummary,
|
|
freshnessEvaluation: $freshnessEvaluation,
|
|
scheduleFollowUp: $scheduleFollowUp,
|
|
healthyClaimAllowed: false,
|
|
primaryActionTarget: BackupHealthActionTarget::backupSetView(
|
|
recordId: (int) $latestBackupSet->getKey(),
|
|
reason: TenantBackupHealthAssessment::REASON_LATEST_BACKUP_STALE,
|
|
),
|
|
positiveClaimBoundary: self::POSITIVE_CLAIM_BOUNDARY,
|
|
);
|
|
}
|
|
|
|
if ($qualitySummary?->hasDegradations() === true) {
|
|
return new TenantBackupHealthAssessment(
|
|
tenantId: $tenantId,
|
|
posture: TenantBackupHealthAssessment::POSTURE_DEGRADED,
|
|
primaryReason: TenantBackupHealthAssessment::REASON_LATEST_BACKUP_DEGRADED,
|
|
headline: 'Latest backup is degraded.',
|
|
supportingMessage: $this->combineMessages([
|
|
$qualitySummary->summaryMessage,
|
|
$this->latestBackupAgeMessage($latestBackupSet->completed_at, $now),
|
|
$scheduleFollowUp->summaryMessage,
|
|
]),
|
|
latestRelevantBackupSetId: (int) $latestBackupSet->getKey(),
|
|
latestRelevantCompletedAt: $latestBackupSet->completed_at,
|
|
qualitySummary: $qualitySummary,
|
|
freshnessEvaluation: $freshnessEvaluation,
|
|
scheduleFollowUp: $scheduleFollowUp,
|
|
healthyClaimAllowed: false,
|
|
primaryActionTarget: BackupHealthActionTarget::backupSetView(
|
|
recordId: (int) $latestBackupSet->getKey(),
|
|
reason: TenantBackupHealthAssessment::REASON_LATEST_BACKUP_DEGRADED,
|
|
),
|
|
positiveClaimBoundary: self::POSITIVE_CLAIM_BOUNDARY,
|
|
);
|
|
}
|
|
|
|
$scheduleNeedsFollowUp = $scheduleFollowUp->needsFollowUp;
|
|
|
|
return new TenantBackupHealthAssessment(
|
|
tenantId: $tenantId,
|
|
posture: TenantBackupHealthAssessment::POSTURE_HEALTHY,
|
|
primaryReason: $scheduleNeedsFollowUp ? TenantBackupHealthAssessment::REASON_SCHEDULE_FOLLOW_UP : null,
|
|
headline: $scheduleNeedsFollowUp
|
|
? 'Backup schedules need follow-up.'
|
|
: 'Backups are recent and healthy.',
|
|
supportingMessage: $scheduleNeedsFollowUp
|
|
? $this->combineMessages([
|
|
'The latest completed backup is recent and shows no material degradation.',
|
|
$scheduleFollowUp->summaryMessage,
|
|
])
|
|
: $this->latestBackupAgeMessage($latestBackupSet->completed_at, $now),
|
|
latestRelevantBackupSetId: (int) $latestBackupSet->getKey(),
|
|
latestRelevantCompletedAt: $latestBackupSet->completed_at,
|
|
qualitySummary: $qualitySummary,
|
|
freshnessEvaluation: $freshnessEvaluation,
|
|
scheduleFollowUp: $scheduleFollowUp,
|
|
healthyClaimAllowed: ! $scheduleNeedsFollowUp,
|
|
primaryActionTarget: $scheduleNeedsFollowUp
|
|
? BackupHealthActionTarget::backupSchedulesIndex(TenantBackupHealthAssessment::REASON_SCHEDULE_FOLLOW_UP)
|
|
: null,
|
|
positiveClaimBoundary: self::POSITIVE_CLAIM_BOUNDARY,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @param iterable<int, Tenant|int> $tenants
|
|
* @return array<int, int>
|
|
*/
|
|
private function normalizeTenantIds(iterable $tenants): array
|
|
{
|
|
$tenantIds = [];
|
|
|
|
foreach ($tenants as $tenant) {
|
|
$tenantId = $tenant instanceof Tenant
|
|
? (int) $tenant->getKey()
|
|
: (int) $tenant;
|
|
|
|
if ($tenantId > 0) {
|
|
$tenantIds[] = $tenantId;
|
|
}
|
|
}
|
|
|
|
return array_values(array_unique($tenantIds));
|
|
}
|
|
|
|
private function latestBackupAgeMessage(?CarbonInterface $completedAt, CarbonImmutable $now): ?string
|
|
{
|
|
if (! $completedAt instanceof CarbonInterface) {
|
|
return null;
|
|
}
|
|
|
|
return sprintf(
|
|
'The latest completed backup was %s.',
|
|
$completedAt->diffForHumans($now, [
|
|
'parts' => 2,
|
|
'join' => true,
|
|
'short' => false,
|
|
'syntax' => CarbonInterface::DIFF_RELATIVE_TO_NOW,
|
|
]),
|
|
);
|
|
}
|
|
|
|
private function freshnessHours(): int
|
|
{
|
|
return max(1, (int) config('tenantpilot.backup_health.freshness_hours', 24));
|
|
}
|
|
|
|
private function scheduleOverdueGraceMinutes(): int
|
|
{
|
|
return max(1, (int) config('tenantpilot.backup_health.schedule_overdue_grace_minutes', 30));
|
|
}
|
|
|
|
private function scheduleSummaryMessage(
|
|
int $enabledScheduleCount,
|
|
int $overdueScheduleCount,
|
|
int $failedRecentRunCount,
|
|
int $neverSuccessfulCount,
|
|
): ?string {
|
|
if ($enabledScheduleCount === 0) {
|
|
return null;
|
|
}
|
|
|
|
if ($neverSuccessfulCount > 0) {
|
|
return $neverSuccessfulCount === 1
|
|
? 'One enabled schedule has not produced a successful run yet.'
|
|
: sprintf('%d enabled schedules have not produced a successful run yet.', $neverSuccessfulCount);
|
|
}
|
|
|
|
if ($overdueScheduleCount > 0) {
|
|
return $overdueScheduleCount === 1
|
|
? 'One enabled schedule looks overdue.'
|
|
: sprintf('%d enabled schedules look overdue.', $overdueScheduleCount);
|
|
}
|
|
|
|
if ($failedRecentRunCount > 0) {
|
|
return $failedRecentRunCount === 1
|
|
? 'One enabled schedule needs follow-up after the last run.'
|
|
: sprintf('%d enabled schedules need follow-up after the last run.', $failedRecentRunCount);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* @param array<int, string|null> $messages
|
|
*/
|
|
private function combineMessages(array $messages): ?string
|
|
{
|
|
$parts = array_values(array_filter(
|
|
Arr::flatten($messages),
|
|
static fn (mixed $message): bool => is_string($message) && $message !== ''
|
|
));
|
|
|
|
if ($parts === []) {
|
|
return null;
|
|
}
|
|
|
|
return implode(' ', $parts);
|
|
}
|
|
}
|